101 lines
4.6 KiB
OCaml
101 lines
4.6 KiB
OCaml
open Logarion
|
|
|
|
(*TODO: move to converters (style, feed checks)*)
|
|
let is_older s d = try Unix.((stat d).st_mtime < (stat s).st_mtime) with _-> true
|
|
|
|
let convert cs r (text, files) = match Text.str "Content-Type" text with
|
|
| "" | "text/plain" ->
|
|
let source = List.hd files in
|
|
let dest = Filename.concat r.Conversion.dir (Text.short_id text) in
|
|
List.fold_left
|
|
(fun a f ->
|
|
match f.Conversion.page with None -> false || a
|
|
| Some page ->
|
|
let dest = dest ^ f.Conversion.ext in
|
|
(if is_older source dest then (File_store.file dest (page r text); true) else false)
|
|
|| a)
|
|
false cs
|
|
| x -> Printf.eprintf "Can't convert Content-Type: %s file: %s" x text.Text.title; false
|
|
|
|
let converters types kv =
|
|
let n = String.split_on_char ',' types in
|
|
let t = [] in
|
|
let t = if List.(mem "all" n || mem "htm" n) then (Html.converter kv)::t else t in
|
|
let t = if List.(mem "all" n || mem "atom" n) then (Atom.converter "text/html")::t else t in
|
|
let t = if List.(mem "all" n || mem "gmi" n) then (Gemini.converter)::t else t in
|
|
let t = if List.(mem "all" n || mem "gmi-atom" n) then (Atom.converter "text/gemini")::t else t in
|
|
t
|
|
|
|
let acc_ref id t a =
|
|
Conversion.Id_map.update t (function
|
|
| Some s -> Some (Conversion.Ref_set.add id s)
|
|
| None -> Some (Conversion.Ref_set.singleton id)
|
|
) a
|
|
|
|
let fold_refs text refs = String_set.fold (acc_ref text.Text.id) (Text.set "references" text) refs
|
|
let fold_reps text reps = String_set.fold (acc_ref text.Text.id) (Text.set "in-reply-to" text) reps
|
|
|
|
let directory converters noindex repo =
|
|
let fn (ts,refs,reps,ls,acc) ((elt,_) as r) =
|
|
Topic_set.to_map ts (Text.set "topics" elt),
|
|
fold_refs elt refs, fold_reps elt reps,
|
|
elt::ls,
|
|
if convert converters {repo with references = refs; replies = reps} r then acc+1 else acc in
|
|
let topics, references, replies, texts, count =
|
|
File_store.(fold ~dir:repo.Conversion.dir ~order:oldest fn
|
|
(Topic_set.Map.empty, Conversion.Id_map.empty, Conversion.Id_map.empty, [], 0)) in
|
|
let topic_roots = try List.rev @@ String_set.list_of_csv (Store.KV.find "Topics" repo.kv)
|
|
with Not_found -> Topic_set.roots topics in
|
|
Printf.eprintf "%d\n" (Conversion.Id_map.cardinal replies);
|
|
let repo = Conversion.{ repo with
|
|
topic_roots; topics; references; replies; texts = List.rev texts } in
|
|
if not noindex then
|
|
List.iter (fun c -> match c.Conversion.indices with None -> () | Some f -> f repo) converters;
|
|
Printf.printf "Converted: %d Indexed: %d\n" count (List.length texts)
|
|
|
|
let load_kv dir =
|
|
let kv = File_store.of_kv_file () in
|
|
let idx = Filename.concat dir "index.pck" in
|
|
if not (Sys.file_exists idx) then kv else
|
|
match Header_pack.of_string @@ File_store.to_string (idx) with
|
|
| Error s -> prerr_endline s; kv
|
|
| Ok { info; peers; _ } ->
|
|
let kv = if Store.KV.mem "Id" kv then kv else Store.KV.add "Id" info.Header_pack.id kv in
|
|
let kv = if Store.KV.mem "Title" kv then kv else Store.KV.add "Title" info.Header_pack.title kv in
|
|
let kv = if Store.KV.mem "Locations" kv then kv else Store.KV.add "Locations" (String.concat ";\n" info.Header_pack.locations) kv in
|
|
let kv = Store.KV.add "Peers" (String.concat ";\n" Header_pack.(to_str_list peers)) kv in
|
|
kv
|
|
|
|
let at_path types noindex path = match path with
|
|
| "" -> prerr_endline "unspecified text file or directory"
|
|
| path when Sys.file_exists path ->
|
|
if Sys.is_directory path then (
|
|
let kv = load_kv path in
|
|
let repo = { (Conversion.empty ()) with dir = path; kv } in
|
|
directory (converters types kv) noindex repo
|
|
) else (
|
|
match File_store.to_text path with
|
|
| Error s -> prerr_endline s
|
|
| Ok text ->
|
|
let dir = "." in
|
|
let references, replies = File_store.(fold ~dir ~order:newest
|
|
(fun (refs, reps) (elt, _) -> fold_refs elt refs, fold_reps elt reps)
|
|
(Conversion.Id_map.empty, Conversion.Id_map.empty)) in
|
|
let repo = { (Conversion.empty ()) with dir; kv = load_kv ""; references; replies } in
|
|
ignore @@ convert (converters types repo.kv) repo (text, [path])
|
|
)
|
|
| path -> Printf.eprintf "Path doesn't exist: %s" path
|
|
|
|
open Cmdliner
|
|
let term =
|
|
let path = Arg.(value & pos 0 string "" & info [] ~docv:"path"
|
|
~doc:"Text file or directory to convert. If directory is provided, it must contain an index.pck (see: txt index)") in
|
|
let types = Arg.(value & opt string "all" & info ["t"; "type"] ~docv:"output type"
|
|
~doc:"Convert to file type") in
|
|
let noindex = Arg.(value & flag & info ["noindex"]
|
|
~doc:"Don't create indices in target format") in
|
|
Term.(const at_path $ types $ noindex $ path),
|
|
Term.info "convert" ~doc:"convert texts"
|
|
~man:[ `S "DESCRIPTION"; `P "Convert text or indexed texts within a directory to another format.
|
|
If path is a directory must contain an index.pck. Run `txt index` first." ]
|