blob: 54240e1ee42a281293ad136e86886226b263e239 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
|
module Toks where
import Data.Char
import Data.Function
import Data.List
type Tok = String
escape :: Char -> String
escape '\\' = "\\\\"
escape '\n' = "\\n"
escape x = pure x
unescape :: String -> String
unescape [] = []
unescape ('\\':'\\':xs) = '\\' : unescape xs
unescape ('\\':'n':xs) = '\n' : unescape xs
unescape ('\\':_) = error "bad escape on input"
unescape (x:xs) = x : unescape xs
markSpace :: String -> Tok
markSpace [] = error "empty token"
markSpace s@(c:_)
| isSpace c = '.' : s
| otherwise = '|' : s
unmarkSpace :: Tok -> String
unmarkSpace ('.':s) = s
unmarkSpace ('|':s) = s
unmarkSpace _ = error "bad space marking on input"
space :: Tok -> Bool
space ('.':_) = True
space _ = False
joinSpaces :: [Tok] -> [Tok]
joinSpaces [] = []
joinSpaces (a@('.':as):xs) =
case joinSpaces xs of
(('.':bs):xs') -> ('.' : (as ++ bs)) : xs'
xs' -> a : xs'
joinSpaces (x:xs) = x : joinSpaces xs
splitCategory :: String -> [Tok]
splitCategory = make . groupBy ((==) `on` generalCategory)
simpleCategory :: Char -> Int
simpleCategory c
| isSpace c = 0
| isAlpha c = 1
| isNumber c = 2
| otherwise = 3
splitSimple :: String -> [Tok]
splitSimple = make . groupBy ((==) `on` simpleCategory)
make :: [String] -> [Tok]
make = map (concatMap escape . markSpace)
glue :: [String] -> String
glue = concatMap (unmarkSpace . unescape)
fromFile :: String -> [Tok]
fromFile = lines
toFile :: [Tok] -> String
toFile = unlines
|
