1 files changed, 33 insertions, 0 deletions
diff --git a/Toks.hs b/Toks.hs
new file mode 100644
index 0000000..c8ec567
--- /dev/null
+++ b/Toks.hs
@@ -0,0 +1,33 @@
+module Toks where
+
+import Data.Char
+import Data.Function
+import Data.List
+
+escape '\\' = "\\\\"
+escape '\n' = "\\n"
+escape x = pure x
+
+unescape [] = []
+unescape ('\\':'\\':xs) = '\\' : unescape xs
+unescape ('\\':'n':xs) = '\n' : unescape xs
+unescape ('\\':_) = error "bad escape?"
+unescape (x:xs) = x : unescape xs
+
+markSpace [] = error "wat"
+markSpace s@(c:_)
+  | isSpace c = '.' : s
+  | otherwise = '|' : s
+
+unmarkSpace ('.':s) = s
+unmarkSpace ('|':s) = s
+unmarkSpace _ = error "wat"
+
+split =
+  unlines
+    . map (concatMap escape . markSpace)
+    . groupBy ((==) `on` generalCategory)
+
+glueToks = concatMap (unmarkSpace . unescape)
+
+glue = glueToks . lines