@logseq/cli
Advanced tools
+2
-2
| { | ||
| "name": "@logseq/cli", | ||
| "version": "0.4.1", | ||
| "version": "0.4.2", | ||
| "description": "Logseq CLI", | ||
@@ -13,3 +13,3 @@ "bin": { | ||
| "dependencies": { | ||
| "@logseq/nbb-logseq": "github:logseq/nbb-logseq#feat-db-v29", | ||
| "@logseq/nbb-logseq": "github:logseq/nbb-logseq#feat-db-v30", | ||
| "@modelcontextprotocol/sdk": "^1.17.5", | ||
@@ -16,0 +16,0 @@ "better-sqlite3": "~11.10.0", |
+45
-31
@@ -27,3 +27,3 @@ (ns logseq.cli | ||
| (declare table) | ||
| (defn- help [_m] | ||
| (defn- print-general-help [_m] | ||
| (println (str "Usage: logseq [command] [options]\n\nOptions:\n" | ||
@@ -41,3 +41,3 @@ (cli/format-opts {:spec default-spec}))) | ||
| (aget "version"))))) | ||
| (help m))) | ||
| (print-general-help m))) | ||
@@ -55,20 +55,26 @@ (defn- print-command-help [command cmd-map] | ||
| (defn- help-command [{{:keys [command]} :opts}] | ||
| (defn- help-command [{{:keys [command help]} :opts}] | ||
| (if-let [cmd-map (and command (some #(when (= command (first (:cmds %))) %) table))] | ||
| (print-command-help command cmd-map) | ||
| (println "Command" (pr-str command) "does not exist"))) | ||
| ;; handle help --help | ||
| (if-let [cmd-map (and help (some #(when (= "help" (first (:cmds %))) %) table))] | ||
| (print-command-help "help" cmd-map) | ||
| (println "Command" (pr-str command) "does not exist")))) | ||
| (defn- lazy-load-fn | ||
| "Lazy load fn to speed up start time. After nbb requires ~30 namespaces, start time gets close to 1s" | ||
| "Lazy load fn to speed up start time. After nbb requires ~30 namespaces, start time gets close to 1s. | ||
| Also handles --help on all commands" | ||
| [fn-sym] | ||
| (fn [& args] | ||
| (-> (p/let [_ (require (symbol (namespace fn-sym)))] | ||
| (apply (resolve fn-sym) args)) | ||
| (p/catch (fn [err] | ||
| (if (= :sci/error (:type (ex-data err))) | ||
| (nbb.error/print-error-report err) | ||
| (js/console.error "Error:" err)) | ||
| (js/process.exit 1)))))) | ||
| (if (get-in (first args) [:opts :help]) | ||
| (help-command {:opts {:command (-> args first :dispatch first)}}) | ||
| (-> (p/let [_ (require (symbol (namespace fn-sym)))] | ||
| (apply (resolve fn-sym) args)) | ||
| (p/catch (fn [err] | ||
| (if (= :sci/error (:type (ex-data err))) | ||
| (nbb.error/print-error-report err) | ||
| (js/console.error "Error:" err)) | ||
| (js/process.exit 1))))))) | ||
| (def ^:private table | ||
| (def ^:private table* | ||
| [{:cmds ["list"] :desc "List local graphs" | ||
@@ -122,27 +128,35 @@ :fn (lazy-load-fn 'logseq.cli.commands.graph/list-graphs)} | ||
| (defn- error-if-db-version-not-installed | ||
| ;; Spec shared with all commands | ||
| (def ^:private shared-spec | ||
| {:help {:alias :h | ||
| :desc "Print help"}}) | ||
| (def ^:private table | ||
| (mapv (fn [m] (update m :spec #(merge % shared-spec))) table*)) | ||
| (defn- warn-if-db-version-not-installed | ||
| [] | ||
| (when-not (fs/existsSync (cli-common-graph/get-db-graphs-dir)) | ||
| (println "Error: The database version's desktop app is not installed. Please install per https://github.com/logseq/logseq/#-database-version.") | ||
| (js/process.exit 1))) | ||
| (println "[WARN] The database version's desktop app is not installed. Please install per https://github.com/logseq/logseq/#-database-version."))) | ||
| (defn ^:api -main [& args] | ||
| (when-not (contains? #{nil "-h" "--help"} (first args)) | ||
| (error-if-db-version-not-installed)) | ||
| (warn-if-db-version-not-installed) | ||
| (try | ||
| (cli/dispatch table | ||
| args | ||
| {:error-fn (fn [{:keys [cause msg option] type' :type :as data}] | ||
| (if (and (= :org.babashka/cli type') | ||
| (= :require cause)) | ||
| (do | ||
| (println "Error: Command missing required" | ||
| (if (get-in data [:spec option]) "option" "argument") | ||
| (pr-str (name option))) | ||
| (when-let [cmd-m (some #(when (= {:spec (:spec %) | ||
| :require (:require %)} | ||
| (select-keys data [:spec :require])) %) table)] | ||
| (print-command-help (-> cmd-m :cmds first) cmd-m))) | ||
| (throw (ex-info msg data))) | ||
| (js/process.exit 1))}) | ||
| {:error-fn (fn [{:keys [cause msg option opts] type' :type :as data}] | ||
| ;; Options aren't required when printing help | ||
| (when-not (:help opts) | ||
| (if (and (= :org.babashka/cli type') | ||
| (= :require cause)) | ||
| (do | ||
| (println "Error: Command missing required" | ||
| (if (get-in data [:spec option]) "option" "argument") | ||
| (pr-str (name option))) | ||
| (when-let [cmd-m (some #(when (= {:spec (:spec %) | ||
| :require (:require %)} | ||
| (select-keys data [:spec :require])) %) table)] | ||
| (print-command-help (-> cmd-m :cmds first) cmd-m))) | ||
| (throw (ex-info msg data))) | ||
| (js/process.exit 1)))}) | ||
| (catch ^:sci/error js/Error e | ||
@@ -149,0 +163,0 @@ (nbb.error/print-error-report e) |
@@ -22,3 +22,12 @@ (ns logseq.cli.commands.export-edn | ||
| (defn- local-export [{{:keys [graph] :as options} :opts}] | ||
| (defn- validate-export | ||
| [export-map {:keys [catch-validation-errors?]}] | ||
| (println "Validating export which can take awhile on large graphs ...") | ||
| (if-let [error (:error (sqlite-export/validate-export export-map))] | ||
| (if catch-validation-errors? | ||
| (js/console.error error) | ||
| (cli-util/error error)) | ||
| (println "Valid export!"))) | ||
| (defn- local-export [{{:keys [graph validate] :as options} :opts}] | ||
| (when-not graph | ||
@@ -30,2 +39,4 @@ (cli-util/error "Command missing required option 'graph'")) | ||
| export-map (sqlite-export/build-export @conn (build-export-options options))] | ||
| (when validate | ||
| (validate-export export-map options)) | ||
| (write-export-edn-map export-map options)) | ||
@@ -35,3 +46,3 @@ (cli-util/error "Graph" (pr-str graph) "does not exist"))) | ||
| (defn- api-export | ||
| [{{:keys [api-server-token] :as options} :opts}] | ||
| [{{:keys [api-server-token validate] :as options} :opts}] | ||
| (let [opts (build-export-options options)] | ||
@@ -42,2 +53,4 @@ (-> (p/let [resp (cli-util/api-fetch api-server-token "logseq.cli.export_edn" [(clj->js opts)])] | ||
| export-map (sqlite-util/transit-read (aget body "export-body"))] | ||
| (when validate | ||
| (validate-export export-map options)) | ||
| (write-export-edn-map export-map (assoc options :graph (.-graph body)))) | ||
@@ -44,0 +57,0 @@ (cli-util/api-handle-error-response resp))) |
@@ -10,2 +10,3 @@ (ns logseq.cli.commands.import-edn | ||
| [logseq.db.sqlite.util :as sqlite-util] | ||
| [logseq.outliner.db-pipeline :as db-pipeline] | ||
| [promesa.core :as p])) | ||
@@ -28,2 +29,3 @@ | ||
| (let [conn (apply sqlite-cli/open-db! (cli-util/->open-db-args graph)) | ||
| _ (db-pipeline/add-listener conn) | ||
| _ (cli-util/ensure-db-graph-for-command @conn) | ||
@@ -30,0 +32,0 @@ {:keys [init-tx block-props-tx misc-tx]} |
@@ -10,2 +10,3 @@ (ns logseq.cli.commands.mcp-server | ||
| [logseq.db.common.sqlite-cli :as sqlite-cli] | ||
| [logseq.outliner.db-pipeline :as db-pipeline] | ||
| [nbb.core :as nbb] | ||
@@ -87,3 +88,4 @@ [promesa.core :as p])) | ||
| (let [mcp-server (cli-common-mcp-server/create-mcp-server) | ||
| conn (apply sqlite-cli/open-db! (cli-util/->open-db-args graph))] | ||
| conn (apply sqlite-cli/open-db! (cli-util/->open-db-args graph)) | ||
| _ (db-pipeline/add-listener conn)] | ||
| (doseq [[k v] local-tools] | ||
@@ -99,3 +101,4 @@ (.registerTool mcp-server (name k) (:config v) (partial (:fn v) conn))) | ||
| (if-let [tool-m (get local-tools debug-tool)] | ||
| (let [conn (apply sqlite-cli/open-db! (cli-util/->open-db-args graph))] | ||
| (let [conn (apply sqlite-cli/open-db! (cli-util/->open-db-args graph)) | ||
| _ (db-pipeline/add-listener conn)] | ||
| (p/let [resp ((:fn tool-m) conn (clj->js (dissoc opts :debug-tool)))] | ||
@@ -102,0 +105,0 @@ (js/console.log (clj->js resp)))) |
@@ -39,5 +39,5 @@ (ns logseq.cli.commands.search | ||
| (println (string/join "\n" | ||
| (->> results | ||
| (map #(string/replace % "\n" "\\\\n")) | ||
| (map highlight-fn))))))) | ||
| (->> results | ||
| (map #(string/replace % "\n" "\\\\n")) | ||
| (map highlight-fn))))))) | ||
@@ -44,0 +44,0 @@ (defn- api-search |
@@ -5,40 +5,18 @@ (ns logseq.cli.commands.validate | ||
| [cljs.pprint :as pprint] | ||
| [datascript.core :as d] | ||
| [logseq.cli.util :as cli-util] | ||
| [logseq.db.common.sqlite-cli :as sqlite-cli] | ||
| [logseq.db.frontend.malli-schema :as db-malli-schema] | ||
| [logseq.db.frontend.validate :as db-validate] | ||
| [malli.error :as me])) | ||
| [logseq.db.frontend.validate :as db-validate])) | ||
| (defn- validate-db* | ||
| "Validate datascript db as a vec of entity maps" | ||
| [db ent-maps* {:keys [open-schema]}] | ||
| (let [ent-maps (db-malli-schema/update-properties-in-ents db ent-maps*) | ||
| explainer (db-validate/get-schema-explainer (not open-schema))] | ||
| (if-let [explanation (binding [db-malli-schema/*db-for-validate-fns* db | ||
| db-malli-schema/*closed-values-validate?* true] | ||
| (->> (map (fn [e] (dissoc e :db/id)) ent-maps) explainer not-empty))] | ||
| (let [ent-errors | ||
| (->> (db-validate/group-errors-by-entity db ent-maps (:errors explanation)) | ||
| (map #(update % :errors | ||
| (fn [errs] | ||
| ;; errs looks like: {178 {:logseq.property/hide? ["disallowed key"]}} | ||
| ;; map is indexed by :in which is unused since all errors are for the same map | ||
| (->> (me/humanize {:errors errs}) | ||
| vals | ||
| (apply merge-with into))))))] | ||
| (println "Found" (count ent-errors) | ||
| (if (= 1 (count ent-errors)) "entity" "entities") | ||
| "with errors:") | ||
| (pprint/pprint ent-errors) | ||
| (js/process.exit 1)) | ||
| (println "Valid!")))) | ||
| (defn- validate-db [db db-name options] | ||
| (let [datoms (d/datoms db :eavt) | ||
| ent-maps (db-malli-schema/datoms->entities datoms)] | ||
| (println "Read graph" (str db-name " with counts: " | ||
| (pr-str (assoc (db-validate/graph-counts db ent-maps) | ||
| :datoms (count datoms))))) | ||
| (validate-db* db ent-maps options))) | ||
| (if-let [errors (:errors | ||
| (db-validate/validate-local-db! | ||
| db | ||
| (merge options {:db-name db-name :verbose true})))] | ||
| (do | ||
| (println "Found" (count errors) | ||
| (if (= 1 (count errors)) "entity" "entities") | ||
| "with errors:") | ||
| (pprint/pprint errors) | ||
| (js/process.exit 1)) | ||
| (println "Valid!"))) | ||
@@ -45,0 +23,0 @@ (defn- validate-graph [graph options] |
@@ -129,4 +129,3 @@ (ns logseq.cli.common.mcp.tools | ||
| (throw (ex-info (str "Error while building import data: " error) {}))) | ||
| (let [tx-meta {::sqlite-export/imported-data? true | ||
| :import-db? true}] | ||
| (let [tx-meta {::sqlite-export/imported-data? true}] | ||
| (ldb/transact! conn (vec (concat init-tx block-props-tx misc-tx)) tx-meta)))) | ||
@@ -133,0 +132,0 @@ |
@@ -20,4 +20,2 @@ (ns logseq.cli.spec | ||
| :desc "File to save export"} | ||
| :catch-validation-errors? {:alias :c | ||
| :desc "Catch validation errors for dev"} | ||
| :exclude-namespaces {:alias :e | ||
@@ -33,3 +31,7 @@ :coerce #{} | ||
| :desc "Export type" | ||
| :default :graph}}) | ||
| :default :graph} | ||
| :validate {:alias :v | ||
| :desc "(Dev) Validate export with a temp graph built on exported EDN"} | ||
| :catch-validation-errors? {:alias :c | ||
| :desc "(Dev) Catch validation errors and still write invalid EDN"}}) | ||
@@ -36,0 +38,0 @@ (def import-edn |
@@ -62,3 +62,2 @@ (ns logseq.common.config | ||
| (defn remove-asset-protocol | ||
@@ -65,0 +64,0 @@ [s] |
@@ -106,13 +106,14 @@ (ns logseq.db | ||
| (defn- transact-sync | ||
| [repo-or-conn tx-data tx-meta] | ||
| [conn tx-data tx-meta] | ||
| (try | ||
| (let [conn repo-or-conn | ||
| db @conn | ||
| (let [db @conn | ||
| db-based? (entity-plus/db-based-graph? db)] | ||
| (if (and db-based? | ||
| (not (:reset-conn! tx-meta)) | ||
| (not (:initial-db? tx-meta)) | ||
| (not (:rtc-download-graph? tx-meta)) | ||
| (not (:skip-validate-db? tx-meta false)) | ||
| (not (:logseq.graph-parser.exporter/new-graph? tx-meta))) | ||
| (not | ||
| (or (:batch-temp-conn? @conn) | ||
| (:rtc-download-graph? tx-meta) | ||
| (:reset-conn! tx-meta) | ||
| (:initial-db? tx-meta) | ||
| (:skip-validate-db? tx-meta false) | ||
| (:logseq.graph-parser.exporter/new-graph? tx-meta)))) | ||
| (let [tx-report* (d/with db tx-data tx-meta) | ||
@@ -136,4 +137,9 @@ pipeline-f @*transact-pipeline-fn | ||
| (f tx-report errors)) | ||
| (throw (ex-info "DB write failed with invalid data" {:tx-data tx-data | ||
| :pipeline-tx-data (:tx-data tx-report)})))) | ||
| (throw (ex-info "DB write failed with invalid data" {:tx-meta tx-meta | ||
| :tx-data tx-data | ||
| :errors errors | ||
| :pipeline-tx-data (map | ||
| (fn [[e a v t]] | ||
| [e a v t]) | ||
| (:tx-data tx-report))})))) | ||
| tx-report) | ||
@@ -169,3 +175,3 @@ (d/transact! conn tx-data tx-meta))) | ||
| (delete-blocks/update-refs-history-and-macros @repo-or-conn tx-data tx-meta)) | ||
| tx-data (distinct (concat tx-data delete-blocks-tx))] | ||
| tx-data (concat tx-data delete-blocks-tx)] | ||
@@ -184,2 +190,24 @@ ;; Ensure worker can handle the request sequentially (one by one) | ||
| (defn transact-with-temp-conn! | ||
| "Validate db and store once for a batch transaction, the `temp` conn can still load data from disk, | ||
| however it can't write to the disk." | ||
| [conn tx-meta batch-tx-fn] | ||
| (let [temp-conn (d/conn-from-db @conn) | ||
| *batch-tx-data (volatile! [])] | ||
| ;; can read from disk, write is disallowed | ||
| (swap! temp-conn assoc | ||
| :skip-store? true | ||
| :batch-temp-conn? true) | ||
| (d/listen! temp-conn ::temp-conn-batch-tx | ||
| (fn [{:keys [tx-data]}] | ||
| (vswap! *batch-tx-data into tx-data))) | ||
| (batch-tx-fn temp-conn) | ||
| (let [tx-data @*batch-tx-data] | ||
| (d/unlisten! temp-conn ::temp-conn-batch-tx) | ||
| (reset! temp-conn nil) | ||
| (vreset! *batch-tx-data nil) | ||
| (when (seq tx-data) | ||
| ;; transact tx-data to `conn` and validate db | ||
| (transact! conn tx-data tx-meta))))) | ||
| (def page? common-entity-util/page?) | ||
@@ -186,0 +214,0 @@ (def internal-page? entity-util/internal-page?) |
@@ -198,2 +198,26 @@ (ns logseq.db.common.entity-plus | ||
| (defn- entity-ish? [x] | ||
| (instance? Entity x)) | ||
| (defn- ->printable | ||
| "Convert values so printing won't recurse forever: | ||
| - Entity => {:db/id eid} | ||
| - coll of Entity => coll of {:db/id ...} | ||
| - maps are walked (rare but safe)" | ||
| [x] | ||
| (cond | ||
| (entity-ish? x) | ||
| {:db/id (.-eid ^Entity x)} | ||
| (map? x) | ||
| (reduce-kv (fn [m k v] (assoc m k (->printable v))) {} x) | ||
| (sequential? x) | ||
| (map ->printable x) | ||
| (set? x) | ||
| (into #{} (map ->printable x)) | ||
| :else x)) | ||
| #?(:org.babashka/nbb | ||
@@ -226,4 +250,7 @@ nil | ||
| (-pr-writer [this writer opts] | ||
| (let [m (-> (into {} (cache-with-kv this)) | ||
| (assoc :db/id (.-eid this)))] | ||
| ;; Touch ONLY this entity, to materialize its forward attrs | ||
| (entity/touch this) | ||
| (let [m0 (into {} (cache-with-kv this)) | ||
| m (-> (reduce-kv (fn [m k v] (assoc m k (->printable v))) {} m0) | ||
| (assoc :db/id (.-eid this)))] | ||
| (-pr-writer m writer opts))) | ||
@@ -230,0 +257,0 @@ |
@@ -89,6 +89,6 @@ (ns logseq.db.common.initial-data | ||
| (defn get-block-children-ids | ||
| "Returns children UUIDs, notice the result doesn't include property value children ids." | ||
| [db block-uuid & {:keys [include-collapsed-children?] | ||
| :or {include-collapsed-children? true}}] | ||
| (when-let [eid (:db/id (d/entity db [:block/uuid block-uuid]))] | ||
| "Returns children ids, notice the result doesn't include property value children ids." | ||
| [db block-eid & {:keys [include-collapsed-children?] | ||
| :or {include-collapsed-children? true}}] | ||
| (when-let [eid (:db/id (d/entity db block-eid))] | ||
| (let [seen (volatile! #{})] | ||
@@ -104,4 +104,4 @@ (loop [eids-to-expand [eid]] | ||
| (:block/_parent e)))) eids-to-expand) | ||
| uuids-to-add (keep :block/uuid children)] | ||
| (vswap! seen (partial apply conj) uuids-to-add) | ||
| ids-to-add (keep :db/id children)] | ||
| (vswap! seen (partial apply conj) ids-to-add) | ||
| (recur (keep :db/id children))))) | ||
@@ -112,12 +112,12 @@ @seen))) | ||
| "Including nested children, notice the result doesn't include property values." | ||
| {:arglists '([db block-uuid & {:keys [include-collapsed-children?]}])} | ||
| [db block-uuid & {:as opts}] | ||
| (let [ids (get-block-children-ids db block-uuid opts)] | ||
| {:arglists '([db eid & {:keys [include-collapsed-children?]}])} | ||
| [db eid & {:as opts}] | ||
| (let [ids (get-block-children-ids db eid opts)] | ||
| (when (seq ids) | ||
| (map (fn [id] (d/entity db [:block/uuid id])) ids)))) | ||
| (map (fn [id] (d/entity db id)) ids)))) | ||
| (defn get-block-full-children-ids | ||
| "Including nested, collapsed and property value children." | ||
| {:arglists '([db block-uuid])} | ||
| [db block-uuid] | ||
| {:arglists '([db block-eid])} | ||
| [db block-eid] | ||
| (d/q | ||
@@ -127,6 +127,5 @@ '[:find [?c ...] | ||
| :where | ||
| [?p :block/uuid ?id] | ||
| (parent ?p ?c)] | ||
| (parent ?id ?c)] | ||
| db | ||
| block-uuid | ||
| block-eid | ||
| (:parent rules/rules))) | ||
@@ -225,3 +224,3 @@ | ||
| (let [children (when children? | ||
| (let [children-blocks (get-block-children db (:block/uuid block) {:include-collapsed-children? include-collapsed-children?}) | ||
| (let [children-blocks (get-block-children db (:db/id block) {:include-collapsed-children? include-collapsed-children?}) | ||
| large-page? (>= (count children-blocks) 100) | ||
@@ -228,0 +227,0 @@ children (let [children' (if large-page? |
@@ -12,2 +12,3 @@ (ns logseq.db.common.reference | ||
| [logseq.db.frontend.class :as db-class] | ||
| [logseq.db.frontend.entity-util :as entity-util] | ||
| [logseq.db.frontend.rules :as rules])) | ||
@@ -99,2 +100,3 @@ | ||
| ;; TODO(perf): recursive datascript rule is still too slow for filters for large graphs | ||
| (defn get-linked-references | ||
@@ -107,24 +109,40 @@ [db id] | ||
| includes (map :db/id (:included page-filters)) | ||
| has-filters? (or (seq excludes) (seq includes)) | ||
| class-ids (when (ldb/class? entity) | ||
| (let [class-children (db-class/get-structured-children db id)] | ||
| (set (conj class-children id)))) | ||
| full-ref-block-ids (->> (mapcat (fn [id] (map :db/id (:block/_refs (d/entity db id)))) ids) | ||
| full-ref-block-ids (->> ids | ||
| (mapcat (fn [id] (:block/_refs (d/entity db id)))) | ||
| (remove (fn [ref] | ||
| (or | ||
| (when class-ids | ||
| (some class-ids (map :db/id (:block/tags ref)))) | ||
| (entity-util/hidden? ref) | ||
| (entity-util/hidden? (:block/page ref))))) | ||
| (map :db/id) | ||
| set) | ||
| matched-ref-block-ids (set (d/q (filter-refs-query includes excludes class-ids) | ||
| db | ||
| (rules/extract-rules rules/db-query-dsl-rules | ||
| [:has-ref] | ||
| {:deps rules/rules-dependencies}) | ||
| ids)) | ||
| matched-refs-with-children-ids (let [*result (atom #{})] | ||
| (doseq [ref-id matched-ref-block-ids] | ||
| (get-block-parents-until-top-ref db id ref-id full-ref-block-ids *result)) | ||
| @*result) | ||
| ref-blocks (->> (set/intersection full-ref-block-ids matched-refs-with-children-ids) | ||
| matched-ref-block-ids (when has-filters? | ||
| (let [ref-ids (d/q (filter-refs-query includes excludes class-ids) | ||
| db | ||
| (rules/extract-rules rules/db-query-dsl-rules | ||
| [:has-ref] | ||
| {:deps rules/rules-dependencies}) | ||
| ids)] | ||
| (set ref-ids))) | ||
| matched-refs-with-children-ids (when has-filters? | ||
| (let [*result (atom #{})] | ||
| (doseq [ref-id matched-ref-block-ids] | ||
| (get-block-parents-until-top-ref db id ref-id full-ref-block-ids *result)) | ||
| @*result)) | ||
| ref-blocks (->> (if has-filters? | ||
| (set/intersection full-ref-block-ids matched-refs-with-children-ids) | ||
| full-ref-block-ids) | ||
| (map (fn [id] (d/entity db id)))) | ||
| filter-exists? (or (seq excludes) (seq includes)) | ||
| children-ids (set (remove full-ref-block-ids matched-refs-with-children-ids))] | ||
| children-ids (if has-filters? | ||
| (set (remove full-ref-block-ids matched-refs-with-children-ids)) | ||
| (->> (mapcat (fn [ref] (ldb/get-block-children-ids db (:db/id ref))) ref-blocks) | ||
| set))] | ||
| {:ref-blocks ref-blocks | ||
| :ref-pages-count (get-ref-pages-count db id ref-blocks children-ids) | ||
| :ref-matched-children-ids (when filter-exists? children-ids)})) | ||
| :ref-matched-children-ids (when has-filters? children-ids)})) | ||
@@ -131,0 +149,0 @@ (defn get-unlinked-references |
@@ -13,7 +13,9 @@ (ns logseq.db.frontend.asset | ||
| (defn <get-file-array-buffer-checksum | ||
| "Given a file's ArrayBuffer, returns its checksum in a promise" | ||
| [file-array-buffer] | ||
| (-> (js/crypto.subtle.digest "SHA-256" file-array-buffer) | ||
| (.then (fn [dig] (js/Uint8Array. dig))) | ||
| (.then decode-digest))) | ||
| "Given a file's ArrayBuffer or String, returns its checksum in a promise" | ||
| [s] | ||
| (let [array-buffer (if (string? s) | ||
| (.encode (js/TextEncoder.) s) s)] | ||
| (-> (js/crypto.subtle.digest "SHA-256" array-buffer) | ||
| (.then (fn [dig] (js/Uint8Array. dig))) | ||
| (.then decode-digest)))) | ||
@@ -20,0 +22,0 @@ (defn asset-path->type |
@@ -306,3 +306,3 @@ (ns logseq.db.frontend.malli-schema | ||
| (concat | ||
| [:map | ||
| [:map {:error/path ["normal-page"]} | ||
| ;; journal-day is only set for journal pages | ||
@@ -319,3 +319,3 @@ [:block/journal-day {:optional true} :int] | ||
| (concat | ||
| [:map | ||
| [:map {:error/path ["class-page"]} | ||
| [:db/ident class-ident] | ||
@@ -327,3 +327,3 @@ [:logseq.property.class/extends [:set :int]]] | ||
| (concat | ||
| [:map | ||
| [:map {:error/path ["class-page"]} | ||
| [:db/ident [:= :logseq.class/Root]]] | ||
@@ -342,3 +342,3 @@ page-attrs | ||
| (concat | ||
| [:map | ||
| [:map {:error/path ["internal-property"]} | ||
| [:db/ident internal-property-ident] | ||
@@ -356,3 +356,3 @@ [:logseq.property/type (apply vector :enum (into db-property-type/internal-built-in-property-types | ||
| (concat | ||
| [:map | ||
| [:map {:error/path ["user-property"]} | ||
| [:db/ident user-property-ident] | ||
@@ -369,3 +369,3 @@ [:logseq.property/type (apply vector :enum (into db-property-type/user-allowed-internal-property-types | ||
| (concat | ||
| [:map | ||
| [:map {:error/path ["plugin-property"]} | ||
| [:db/ident plugin-property-ident] | ||
@@ -399,3 +399,3 @@ [:logseq.property/type (apply vector :enum (concat db-property-type/user-built-in-property-types [:json :string :page]))]] | ||
| (concat | ||
| [:map | ||
| [:map {:error/path ["hidden-page"]} | ||
| ;; pages from :default property uses this but closed-value pages don't | ||
@@ -421,3 +421,3 @@ [:block/order {:optional true} block-order] | ||
| (concat | ||
| [:map] | ||
| [:map {:error/path ["whiteboard-block"]}] | ||
| [[:block/title :string] | ||
@@ -433,3 +433,3 @@ [:block/parent :int] | ||
| (concat | ||
| [:map] | ||
| [:map {:error/path "property-value-block"}] | ||
| [[:logseq.property/value [:or :string :double :boolean]] | ||
@@ -485,3 +485,3 @@ [:logseq.property/created-from-property :int]] | ||
| (concat | ||
| [:map] | ||
| [:map {:error/path ["normal-block"]}] | ||
| block-attrs | ||
@@ -500,3 +500,3 @@ page-or-block-attrs))) | ||
| (concat | ||
| [:map] | ||
| [:map {:error/path ["asset-block"]}] | ||
| ;; TODO: Derive required property types from existing schema in frontend.property | ||
@@ -512,3 +512,3 @@ [[:logseq.property.asset/type :string] | ||
| (def file-block | ||
| [:map | ||
| [:map {:error/path ["file-block"]} | ||
| [:block/uuid :uuid] | ||
@@ -527,3 +527,3 @@ [:block/tx-id {:optional true} :int] | ||
| "A key value map with :db/ident and :kv/value" | ||
| [:map | ||
| [:map {:error/path ["db-ident-key-val"]} | ||
| [:db/ident logseq-ident] | ||
@@ -534,3 +534,3 @@ [:kv/value :any] | ||
| (def property-value-placeholder | ||
| [:map | ||
| [:map {:error/path ["property-value-placeholder"]} | ||
| [:db/ident [:= :logseq.property/empty-placeholder]] | ||
@@ -537,0 +537,0 @@ [:block/uuid :uuid] |
@@ -47,3 +47,3 @@ (ns logseq.db.frontend.property | ||
| :hide? true}} | ||
| :logseq.property/hide? {:title "Hide this property" | ||
| :logseq.property/hide? {:title "Hide this property or page" | ||
| :schema {:type :checkbox | ||
@@ -491,2 +491,12 @@ :hide? true}} | ||
| :queryable? true} | ||
| :logseq.property.asset/external-url {:title "External URL" | ||
| :schema {:type :string | ||
| :hide? false | ||
| :public? true} | ||
| :queryable? true} | ||
| :logseq.property.asset/external-file-name {:title "External file name" | ||
| :schema {:type :string | ||
| :hide? true | ||
| :public? false} | ||
| :queryable? false} | ||
| :logseq.property.asset/size {:title "File Size" | ||
@@ -521,2 +531,4 @@ :schema {:type :raw-number | ||
| :public? false} | ||
| :properties | ||
| {:logseq.property/description "Metadata of asset in remote storage"} | ||
| :rtc property-ignore-rtc} | ||
@@ -523,0 +535,0 @@ :logseq.property.asset/resize-metadata {:title "Asset resize metadata" |
@@ -100,13 +100,28 @@ (ns logseq.db.frontend.property.build | ||
| it should have :original-property-id and :db/ident keys. See | ||
| ->property-value-tx-m for such an example | ||
| ->property-value-tx-m for such an example. Options: | ||
| :pure? - ensure this fn is a pure function" | ||
| [block properties & {:keys [pure?]}] | ||
| * :pure? - ensure this fn is a pure function | ||
| * :pvalue-map? - When set, property value is passed as a map with keys :value and :attributes. This | ||
| allows property values to have additional attributes" | ||
| [block properties & {:keys [pure? pvalue-map?]}] | ||
| ;; Build :db/id out of uuid if block doesn't have one for tx purposes | ||
| (let [block' (if (:db/id block) block (assoc block :db/id [:block/uuid (:block/uuid block)]))] | ||
| (->> properties | ||
| (map (fn [[k v]] | ||
| (let [{:keys [property-value-properties] :as property-map} (if (map? k) k {:db/ident k}) | ||
| (map (fn [[k v*]] | ||
| (let [property-map (if (map? k) k {:db/ident k}) | ||
| gen-uuid-value-prefix (when pure? | ||
| (or (:db/ident block) (:block/uuid block)))] | ||
| (or (:db/ident block) (:block/uuid block))) | ||
| ->pvalue #(if pvalue-map? (:value %) %) | ||
| v (if (set? v*) | ||
| (set (map ->pvalue v*)) | ||
| (->pvalue v*)) | ||
| ->value-block-opts | ||
| (fn ->value-block-opts [v'] | ||
| (cond-> {} | ||
| (and pvalue-map? (seq (:attributes v'))) | ||
| (assoc :properties (:attributes v')) | ||
| pure? | ||
| (assoc :block-uuid | ||
| (common-uuid/gen-uuid :builtin-block-uuid | ||
| (str gen-uuid-value-prefix "-" (->pvalue v'))))))] | ||
| (assert (:db/ident property-map) "Key in map must have a :db/ident") | ||
@@ -118,22 +133,10 @@ (when pure? (assert (some? gen-uuid-value-prefix) block)) | ||
| (set (map #(vector :block/uuid %) v)) | ||
| (set? v) | ||
| (set (map #(build-property-value-block | ||
| block' property-map % | ||
| (cond-> {} | ||
| property-value-properties | ||
| (assoc :properties property-value-properties) | ||
| pure? | ||
| (assoc :block-uuid | ||
| (common-uuid/gen-uuid :builtin-block-uuid (str gen-uuid-value-prefix "-" %))))) | ||
| v)) | ||
| (set? v*) | ||
| (set | ||
| (map #(build-property-value-block block' property-map (->pvalue %) (->value-block-opts %)) | ||
| v*)) | ||
| (uuid? v) | ||
| [:block/uuid v] | ||
| :else | ||
| (build-property-value-block block' property-map v | ||
| (cond-> {} | ||
| property-value-properties | ||
| (assoc :properties property-value-properties) | ||
| pure? | ||
| (assoc :block-uuid | ||
| (common-uuid/gen-uuid :builtin-block-uuid (str gen-uuid-value-prefix "-" v))))))]))) | ||
| (build-property-value-block block' property-map v (->value-block-opts v*)))]))) | ||
| (into {})))) | ||
@@ -140,0 +143,0 @@ |
@@ -40,3 +40,3 @@ (ns logseq.db.frontend.schema | ||
| (def version (parse-schema-version "65.13")) | ||
| (def version (parse-schema-version "65.16")) | ||
@@ -43,0 +43,0 @@ (defn major-version |
@@ -65,3 +65,3 @@ (ns logseq.db.frontend.validate | ||
| (defn group-errors-by-entity | ||
| (defn- group-errors-by-entity | ||
| "Groups malli errors by entities. db is used for providing more debugging info" | ||
@@ -78,4 +78,7 @@ [db ent-maps errors] | ||
| (update :block/page | ||
| (fn [id] (select-keys (d/entity db id) | ||
| [:block/name :block/tags :db/id :block/created-at])))) | ||
| (fn [id] | ||
| (let [page-ent (d/entity db id)] | ||
| (cond-> (select-keys page-ent [:block/name :db/id :block/created-at]) | ||
| (:block/tags page-ent) | ||
| (assoc :block/tags (mapv :db/ident (:block/tags page-ent)))))))) | ||
| :dispatch-key (->> (dissoc ent :db/id) (db-malli-schema/entity-dispatch-key db)) | ||
@@ -119,1 +122,30 @@ :errors errors'}))))) | ||
| :property-pairs (count (mapcat #(-> % db-property/properties (dissoc :block/tags)) entities))})) | ||
| (defn validate-local-db! | ||
| "Validates a local (non-RTC) DB like validate-db! but with default behavior, | ||
| options and logging specific to a local DB. Used by CLI, importer and tests" | ||
| [db & {:keys [db-name open-schema verbose]}] | ||
| (let [datoms (d/datoms db :eavt) | ||
| ent-maps* (db-malli-schema/datoms->entities datoms) | ||
| _ (when verbose | ||
| (println "Read graph" (str db-name " with counts: " | ||
| (pr-str (assoc (graph-counts db ent-maps*) | ||
| :datoms (count datoms)))))) | ||
| ent-maps (mapv | ||
| ;; Remove some UI interactions adding this e.g. import | ||
| #(dissoc % :block.temp/load-status :block.temp/has-children?) | ||
| (db-malli-schema/update-properties-in-ents db ent-maps*)) | ||
| explainer (get-schema-explainer (not open-schema))] | ||
| (when-let [explanation (binding [db-malli-schema/*db-for-validate-fns* db | ||
| db-malli-schema/*closed-values-validate?* true] | ||
| (->> (map (fn [e] (dissoc e :db/id)) ent-maps) explainer not-empty))] | ||
| (let [ent-errors | ||
| (->> (group-errors-by-entity db ent-maps (:errors explanation)) | ||
| (map #(update % :errors | ||
| (fn [errs] | ||
| ;; errs looks like: {178 {:logseq.property/hide? ["disallowed key"]}} | ||
| ;; map is indexed by :in which is unused since all errors are for the same map | ||
| (->> (me/humanize {:errors errs}) | ||
| vals | ||
| (apply merge-with into))))))] | ||
| {:errors ent-errors})))) |
@@ -135,10 +135,3 @@ (ns logseq.db.sqlite.build | ||
| (when-let [property-map (build-property-map-for-pvalue-tx k v new-block properties-config all-idents)] | ||
| [(let [pvalue-attrs (when (:build/property-value v) | ||
| (merge (:build/properties v) | ||
| {:block/tags (mapv #(hash-map :db/ident (get-ident all-idents %)) | ||
| (:build/tags v))} | ||
| (select-keys v [:block/created-at :block/updated-at])))] | ||
| (cond-> property-map | ||
| (and (:build/property-value v) (seq pvalue-attrs)) | ||
| (assoc :property-value-properties pvalue-attrs))) | ||
| [property-map | ||
| (let [property (when (keyword? k) (get properties-config k)) | ||
@@ -148,13 +141,24 @@ closed-value-id (when property (some (fn [item] | ||
| (:uuid item))) | ||
| (get property :build/closed-values)))] | ||
| (cond | ||
| closed-value-id | ||
| closed-value-id | ||
| (get property :build/closed-values))) | ||
| build-pvalue | ||
| (fn build-pvalue [v] | ||
| {:attributes | ||
| (when (:build/property-value v) | ||
| (merge (:build/properties v) | ||
| {:block/tags (mapv #(hash-map :db/ident (get-ident all-idents %)) | ||
| (:build/tags v))} | ||
| (select-keys v [:block/created-at :block/updated-at :block/uuid]))) | ||
| :value | ||
| (cond | ||
| closed-value-id | ||
| closed-value-id | ||
| (:build/property-value v) | ||
| (or (:logseq.property/value v) (:block/title v)) | ||
| (:build/property-value v) | ||
| (or (:logseq.property/value v) (:block/title v)) | ||
| :else | ||
| v))]))) | ||
| (db-property-build/build-property-values-tx-m new-block))) | ||
| :else | ||
| v)})] | ||
| (if (set? v) (set (map build-pvalue v)) (build-pvalue v)))]))) | ||
| ((fn [x] | ||
| (db-property-build/build-property-values-tx-m new-block x {:pvalue-map? true}))))) | ||
@@ -455,3 +459,3 @@ (defn- extract-basic-content-refs | ||
| (defn- build-page-tx [page all-idents page-uuids properties options] | ||
| (defn- build-page-tx [page all-idents page-uuids properties {:keys [build-existing-tx?] :as options}] | ||
| (let [page' (dissoc page :build/tags :build/properties :build/keep-uuid?) | ||
@@ -464,12 +468,14 @@ pvalue-tx-m (->property-value-tx-m page' (:build/properties page) properties all-idents)] | ||
| (conj | ||
| (block-with-timestamps | ||
| (merge | ||
| page' | ||
| (when (seq (:build/properties page)) | ||
| (->block-properties (merge (:build/properties page) (db-property-build/build-properties-with-ref-values pvalue-tx-m)) | ||
| page-uuids all-idents options)) | ||
| (when-let [tag-idents (->> (:build/tags page) (map #(get-ident all-idents %)) seq)] | ||
| {:block/tags (cond-> (mapv #(hash-map :db/ident %) tag-idents) | ||
| (empty? (set/intersection (set tag-idents) db-class/page-classes)) | ||
| (conj :logseq.class/Page))}))))))) | ||
| (merge | ||
| (if build-existing-tx? | ||
| {:block/updated-at (common-util/time-ms)} | ||
| (select-keys (block-with-timestamps page') [:block/created-at :block/updated-at])) | ||
| page' | ||
| (when (seq (:build/properties page)) | ||
| (->block-properties (merge (:build/properties page) (db-property-build/build-properties-with-ref-values pvalue-tx-m)) | ||
| page-uuids all-idents options)) | ||
| (when-let [tag-idents (->> (:build/tags page) (map #(get-ident all-idents %)) seq)] | ||
| {:block/tags (cond-> (mapv #(hash-map :db/ident %) tag-idents) | ||
| (empty? (set/intersection (set tag-idents) db-class/page-classes)) | ||
| (conj :logseq.class/Page))})))))) | ||
@@ -498,5 +504,6 @@ (defn- build-pages-and-blocks-tx | ||
| ;; page tx | ||
| (if build-existing-tx?' | ||
| (if (and build-existing-tx?' (not (:build/properties page')) (not (:build/tags page'))) | ||
| ;; Minimally update existing unless there is useful data to update e.g. properties and tags | ||
| [(select-keys page [:block/uuid :block/created-at :block/updated-at])] | ||
| (build-page-tx page' all-idents page-uuids properties options)) | ||
| (build-page-tx page' all-idents page-uuids properties (assoc options :build-existing-tx? build-existing-tx?'))) | ||
| ;; blocks tx | ||
@@ -503,0 +510,0 @@ (reduce (fn [acc m] |
| (ns logseq.db.sqlite.export | ||
| "Builds sqlite.build EDN to represent nodes in a graph-agnostic way. | ||
| Useful for exporting and importing across DB graphs" | ||
| (:require [clojure.set :as set] | ||
| (:require [cljs.pprint :as pprint] | ||
| [clojure.set :as set] | ||
| [clojure.string :as string] | ||
@@ -18,3 +19,5 @@ [clojure.walk :as walk] | ||
| [logseq.db.frontend.schema :as db-schema] | ||
| [logseq.db.frontend.validate :as db-validate] | ||
| [logseq.db.sqlite.build :as sqlite-build] | ||
| [logseq.db.test.helper :as db-test] | ||
| [medley.core :as medley])) | ||
@@ -56,30 +59,30 @@ | ||
| (defn- build-pvalue-entity-default [db ent-properties pvalue | ||
| {:keys [include-uuid-fn] | ||
| :or {include-uuid-fn (constantly false)} | ||
| {:keys [include-pvalue-uuid-fn] | ||
| :or {include-pvalue-uuid-fn (constantly false)} | ||
| :as options}] | ||
| (if (or (seq ent-properties) (seq (:block/tags pvalue))) | ||
| (cond-> {:build/property-value :block | ||
| :block/title (or (block-title pvalue) | ||
| (:logseq.property/value pvalue))} | ||
| (seq (:block/tags pvalue)) | ||
| (assoc :build/tags (->build-tags (:block/tags pvalue))) | ||
| (let [;; nbb-compatible version of db-property/property-value-content | ||
| property-value-content (or (block-title pvalue) | ||
| (:logseq.property/value pvalue))] | ||
| (if (or (seq ent-properties) (seq (:block/tags pvalue)) (include-pvalue-uuid-fn (:block/uuid pvalue))) | ||
| (cond-> {:build/property-value :block | ||
| :block/title property-value-content} | ||
| (seq (:block/tags pvalue)) | ||
| (assoc :build/tags (->build-tags (:block/tags pvalue))) | ||
| (seq ent-properties) | ||
| (assoc :build/properties | ||
| ;; TODO: Add support for ref properties here and in sqlite.build | ||
| (->> ent-properties | ||
| (keep (fn [[k v]] | ||
| (let [prop-type (:logseq.property/type (d/entity db k))] | ||
| (when-not (contains? db-property-type/all-ref-property-types prop-type) | ||
| [k v])))) | ||
| (into {}))) | ||
| (seq ent-properties) | ||
| (assoc :build/properties | ||
| ;; TODO: Add support for ref properties here and in sqlite.build | ||
| (->> ent-properties | ||
| (keep (fn [[k v]] | ||
| (let [prop-type (:logseq.property/type (d/entity db k))] | ||
| (when-not (contains? db-property-type/all-ref-property-types prop-type) | ||
| [k v])))) | ||
| (into {}))) | ||
| (include-uuid-fn (:block/uuid pvalue)) | ||
| (assoc :block/uuid (:block/uuid pvalue) :build/keep-uuid? true) | ||
| (include-pvalue-uuid-fn (:block/uuid pvalue)) | ||
| (assoc :block/uuid (:block/uuid pvalue) :build/keep-uuid? true) | ||
| (:include-timestamps? options) | ||
| (merge (select-keys pvalue [:block/created-at :block/updated-at]))) | ||
| ;; nbb-compatible version of db-property/property-value-content | ||
| (or (block-title pvalue) | ||
| (:logseq.property/value pvalue)))) | ||
| (:include-timestamps? options) | ||
| (merge (select-keys pvalue [:block/created-at :block/updated-at]))) | ||
| property-value-content))) | ||
@@ -97,6 +100,6 @@ (defonce ignored-properties [:logseq.property/created-by-ref :logseq.property.embedding/hnsw-label-updated-at]) | ||
| (if (contains? #{:node :date} (:logseq.property/type property-ent)) | ||
| ;; Idents take precedence over uuid because they are keep data graph-agnostic | ||
| ;; Idents take precedence over uuid because they keep data graph-agnostic | ||
| (if (:db/ident pvalue) | ||
| (:db/ident pvalue) | ||
| ;; Use metadata distinguish from block references that don't exist like closed values | ||
| ;; Use metadata to distinguish from block references that don't exist like closed values | ||
| ^::existing-property-value? [:block/uuid (:block/uuid pvalue)]) | ||
@@ -202,2 +205,5 @@ (or (:db/ident pvalue) | ||
| (defn block-property-value? [%] | ||
| (and (map? %) (:build/property-value %))) | ||
| (defn- build-node-classes | ||
@@ -208,3 +214,6 @@ [db build-block block-tags properties] | ||
| (mapcat (fn [val-or-vals] | ||
| (mapcat #(when (sqlite-build/page-prop-value? %) (:build/tags (second %))) | ||
| (mapcat #(cond (sqlite-build/page-prop-value? %) | ||
| (:build/tags (second %)) | ||
| (block-property-value? %) | ||
| (:build/tags %)) | ||
| (if (set? val-or-vals) val-or-vals [val-or-vals])))) | ||
@@ -474,3 +483,3 @@ (remove db-class/logseq-class?)) | ||
| {:block/alias (set (map #(vector :block/uuid (:block/uuid %)) (:block/alias page-entity)))}))) | ||
| page-blocks-export {:pages-and-blocks [{:page page :blocks blocks}] | ||
| page-blocks-export {:pages-and-blocks [{:page page :blocks (or blocks [])}] | ||
| :properties properties | ||
@@ -486,4 +495,33 @@ :classes classes}] | ||
| (defn- remove-uuid-if-not-ref-given-uuids | ||
| "Cleans up blocks that have uuids that are not referenced elsewhere. | ||
| Handles a block map and its properties' value blocks (one level deep). For property | ||
| value blocks also handles reverting the value back to its concise form as needed" | ||
| [ref-uuids m] | ||
| (cond-> m | ||
| (not (contains? ref-uuids (:block/uuid m))) | ||
| (dissoc :block/uuid :build/keep-uuid?) | ||
| (:build/properties m) | ||
| (update :build/properties | ||
| (fn [props] | ||
| (let [shrink-property-value | ||
| (fn shrink-property-value [v] | ||
| (if (block-property-value? v) | ||
| ;; Keep property value as map if uuid is referenced or it has unique attributes | ||
| (if (or (contains? ref-uuids (:block/uuid v)) | ||
| ;; Keep this in sync with build-pvalue-entity-default | ||
| ((some-fn :build/tags :build/properties) v)) | ||
| v | ||
| (:block/title v)) | ||
| v))] | ||
| (update-vals props | ||
| (fn [v] | ||
| (if (set? v) | ||
| (set (map shrink-property-value v)) | ||
| (shrink-property-value v))))))))) | ||
| (defn- build-page-export* | ||
| [db eid page-blocks* options] | ||
| "When given the :handle-block-uuids option, handle uuid references between | ||
| blocks including property value blocks" | ||
| [db eid page-blocks* {:keys [handle-block-uuids?] :as options}] | ||
| (let [page-entity (d/entity db eid) | ||
@@ -494,5 +532,33 @@ page-blocks (->> page-blocks* | ||
| (remove :logseq.property/created-from-property)) | ||
| {:keys [pvalue-uuids] :as blocks-export} | ||
| (build-blocks-export db page-blocks options) | ||
| page-blocks-export (build-page-blocks-export db page-entity (merge blocks-export options)) | ||
| {:keys [pvalue-uuids] :as blocks-export*} | ||
| (build-blocks-export db page-blocks (cond-> options | ||
| handle-block-uuids? | ||
| (assoc :include-uuid-fn (constantly true)))) | ||
| blocks-export (if handle-block-uuids? | ||
| (let [remove-uuid-if-not-ref | ||
| (partial remove-uuid-if-not-ref-given-uuids | ||
| (set/union (set pvalue-uuids) | ||
| (when (set? (:include-uuid-fn options)) (:include-uuid-fn options))))] | ||
| (update blocks-export* :blocks #(sqlite-build/update-each-block % remove-uuid-if-not-ref))) | ||
| blocks-export*) | ||
| ontology-page-export | ||
| (when (and (not (:ontology-page? options)) | ||
| (or (entity-util/class? page-entity) (entity-util/property? page-entity))) | ||
| (build-mixed-properties-and-classes-export db [page-entity] {:include-uuid? true})) | ||
| class-page-properties-export | ||
| (when-let [props | ||
| (and (not (:ontology-page? options)) | ||
| (entity-util/class? page-entity) | ||
| (->> (:logseq.property.class/properties page-entity) | ||
| (map :db/ident) | ||
| seq))] | ||
| {:properties (build-export-properties db props {:shallow-copy? true})}) | ||
| page-block-options (cond-> blocks-export | ||
| ontology-page-export | ||
| (merge-export-maps ontology-page-export class-page-properties-export) | ||
| true | ||
| (merge options | ||
| {:blocks (:blocks blocks-export)} | ||
| (when ontology-page-export {:ontology-page? true}))) | ||
| page-blocks-export (build-page-blocks-export db page-entity page-block-options) | ||
| page-block-uuids (set/union pvalue-uuids (:pvalue-uuids page-blocks-export)) | ||
@@ -509,2 +575,4 @@ page-export (assoc page-blocks-export :pvalue-uuids page-block-uuids)] | ||
| (build-page-export* db eid page-blocks* {:include-uuid-fn (:content-ref-uuids content-ref-export) | ||
| :include-pvalue-uuid-fn (:content-ref-uuids content-ref-export) | ||
| :handle-block-uuids? true | ||
| :include-alias? true}) | ||
@@ -652,3 +720,4 @@ page-entity (d/entity db eid) | ||
| (let [page-blocks (get-page-blocks db eid)] | ||
| (build-page-export* db eid page-blocks (merge options {:include-uuid-fn (constantly true)})))) | ||
| (build-page-export* db eid page-blocks (merge options {:include-uuid-fn (constantly true) | ||
| :include-pvalue-uuid-fn (constantly true)})))) | ||
| page-ids) | ||
@@ -660,2 +729,3 @@ ontology-page-exports | ||
| (build-page-export* db eid page-blocks (merge options {:include-uuid-fn (constantly true) | ||
| :include-pvalue-uuid-fn (constantly true) | ||
| :ontology-page? true})))) | ||
@@ -700,5 +770,3 @@ ontology-ids)) | ||
| (defn remove-uuids-if-not-ref [export-map all-ref-uuids] | ||
| (let [remove-uuid-if-not-ref (fn [m] (if (contains? all-ref-uuids (:block/uuid m)) | ||
| m | ||
| (dissoc m :block/uuid :build/keep-uuid?)))] | ||
| (let [remove-uuid-if-not-ref (partial remove-uuid-if-not-ref-given-uuids all-ref-uuids)] | ||
| (-> export-map | ||
@@ -715,3 +783,3 @@ (update :classes update-vals remove-uuid-if-not-ref) | ||
| (walk/postwalk (fn [f] | ||
| (if (and (map? f) (:build/property-value f)) | ||
| (if (block-property-value? f) | ||
| (remove-uuid-if-not-ref f) | ||
@@ -743,3 +811,3 @@ f)) | ||
| "Exports whole graph. Has the following options: | ||
| * :include-timestamps? - When set, timestamps are included on all blocks | ||
| * :include-timestamps? - When set, timestamps are included on all blocks except for property value blocks | ||
| * :exclude-namespaces - A set of parent namespaces to exclude from properties and classes. | ||
@@ -758,3 +826,3 @@ This is useful for graphs seeded with an ontology e.g. schema.org as it eliminates noisy and needless | ||
| (mapcat get-pvalue-uuids (vals (:classes ontology-export))))) | ||
| pages-export (build-graph-pages-export db ontology-export options) | ||
| pages-export (build-graph-pages-export db ontology-export (assoc options :include-pvalue-uuid-fn content-ref-uuids)) | ||
| graph-export* (-> (merge ontology-export pages-export) (dissoc :pvalue-uuids)) | ||
@@ -804,3 +872,3 @@ graph-export (if (seq (:exclude-namespaces options)) | ||
| _ (walk/postwalk (fn [f] | ||
| (if (and (map? f) (:build/property-value f) (:block/uuid f)) | ||
| (if (and (block-property-value? f) (:block/uuid f)) | ||
| (swap! pvalue-known-uuids conj (:block/uuid f)) | ||
@@ -864,5 +932,6 @@ f)) | ||
| (defn- ensure-export-is-valid | ||
| (defn- basic-validate-export | ||
| "Checks that export map is usable by sqlite.build including checking that | ||
| all referenced properties and classes are defined. Checks related to properties and | ||
| all referenced properties and classes are defined. This validation is not as robust | ||
| as validate-export. Checks related to properties and | ||
| classes are disabled when :exclude-namespaces is set because those checks can't be done" | ||
@@ -903,6 +972,6 @@ [db export-map* {:keys [graph-options]}] | ||
| (try | ||
| (ensure-export-is-valid db export-map options) | ||
| (basic-validate-export db export-map options) | ||
| (catch ExceptionInfo e | ||
| (println "Caught error:" e))) | ||
| (ensure-export-is-valid db export-map options)) | ||
| (basic-validate-export db export-map options)) | ||
| (assoc export-map ::export-type export-type))) | ||
@@ -1044,1 +1113,18 @@ | ||
| (sqlite-build/build-blocks-tx (remove-namespaced-keys export-map'')))))) | ||
| (defn validate-export | ||
| "Validates an export by creating an in-memory DB graph, importing the EDN and validating the graph. | ||
| Returns a map with a readable :error key if any error occurs" | ||
| [export-edn] | ||
| (try | ||
| (let [import-conn (db-test/create-conn) | ||
| {:keys [init-tx block-props-tx misc-tx] :as _txs} (build-import export-edn @import-conn {}) | ||
| _ (d/transact! import-conn (concat init-tx block-props-tx misc-tx)) | ||
| validation (db-validate/validate-local-db! @import-conn)] | ||
| (when-let [errors (seq (:errors validation))] | ||
| (js/console.error "Exported EDN has the following invalid errors when imported into a new graph:") | ||
| (pprint/pprint errors) | ||
| {:error (str "The exported EDN has " (count errors) " validation error(s)")})) | ||
| (catch :default e | ||
| (js/console.error "Unexpected export-edn validation error:" e) | ||
| {:error (str "The exported EDN is unexpectedly invalid: " (pr-str (ex-message e)))}))) |
@@ -11,11 +11,9 @@ (ns logseq.graph-parser.mldoc | ||
| :default [lambdaisland.glogi :as log]) | ||
| [goog.object :as gobj] | ||
| [cljs-bean.core :as bean] | ||
| [logseq.graph-parser.utf8 :as utf8] | ||
| [clojure.string :as string] | ||
| [goog.object :as gobj] | ||
| [logseq.common.config :as common-config] | ||
| [logseq.common.util :as common-util] | ||
| [logseq.common.config :as common-config] | ||
| #_:clj-kondo/ignore | ||
| [logseq.graph-parser.schema.mldoc :as mldoc-schema] | ||
| [logseq.db.sqlite.util :as sqlite-util])) | ||
| [logseq.db.sqlite.util :as sqlite-util] | ||
| [logseq.graph-parser.utf8 :as utf8])) | ||
@@ -107,3 +105,3 @@ (defonce parseJson (gobj/get Mldoc "parseJson")) | ||
| (string/triml line))) | ||
| (if remove-first-line? lines r)) | ||
| (if remove-first-line? lines r)) | ||
| content (if remove-first-line? body (cons f body))] | ||
@@ -116,12 +114,12 @@ (string/join "\n" content))) | ||
| (map (fn [[block pos-meta]] | ||
| (if (and (vector? block) | ||
| (= "Src" (first block))) | ||
| (let [{:keys [start_pos end_pos]} pos-meta | ||
| content (utf8/substring content start_pos end_pos) | ||
| spaces (re-find #"^[\t ]+" (first (string/split-lines content))) | ||
| content (if spaces (remove-indentation-spaces content (count spaces) true) | ||
| content) | ||
| block ["Src" (assoc (second block) :full_content content)]] | ||
| [block pos-meta]) | ||
| [block pos-meta])) ast))) | ||
| (if (and (vector? block) | ||
| (= "Src" (first block))) | ||
| (let [{:keys [start_pos end_pos]} pos-meta | ||
| content (utf8/substring content start_pos end_pos) | ||
| spaces (re-find #"^[\t ]+" (first (string/split-lines content))) | ||
| content (if spaces (remove-indentation-spaces content (count spaces) true) | ||
| content) | ||
| block ["Src" (assoc (second block) :full_content content)]] | ||
| [block pos-meta]) | ||
| [block pos-meta])) ast))) | ||
@@ -128,0 +126,0 @@ (defn collect-page-properties |
@@ -412,3 +412,3 @@ (ns logseq.outliner.core | ||
| [:db/retract (:db/id block) :block/page]]) | ||
| (let [ids (cons (:db/id this) (ldb/get-block-full-children-ids db block-id)) | ||
| (let [ids (cons (:db/id this) (ldb/get-block-full-children-ids db (:db/id block))) | ||
| txs (map (fn [id] [:db.fn/retractEntity id]) ids) | ||
@@ -983,3 +983,3 @@ page-tx (let [block (d/entity db [:block/uuid block-id])] | ||
| children-page-tx (when (and not-same-page? (not (ldb/page? block))) | ||
| (let [children-ids (ldb/get-block-full-children-ids db (:block/uuid block))] | ||
| (let [children-ids (ldb/get-block-full-children-ids db (:db/id block))] | ||
| (keep (fn [id] | ||
@@ -986,0 +986,0 @@ (let [child (d/entity db id)] |
@@ -25,3 +25,7 @@ (ns logseq.outliner.page | ||
| [page-entity] | ||
| (let [refs (:block/_refs page-entity) | ||
| (let [refs (->> (:block/_refs page-entity) | ||
| ;; remove child or self that refed this page | ||
| (remove (fn [ref] | ||
| (or (= (:db/id ref) (:db/id page-entity)) | ||
| (= (:db/id (:block/page ref)) (:db/id page-entity)))))) | ||
| id-ref->page #(db-content/content-id-ref->page % [page-entity])] | ||
@@ -28,0 +32,0 @@ (when (seq refs) |
Sorry, the diff of this file is too big to display
GitHub dependency
Supply chain riskContains a dependency which resolves to a GitHub URL. Dependencies fetched from GitHub specifiers are not immutable can be used to inject untrusted code or reduce the likelihood of a reproducible install.
Found 1 instance in 1 package
GitHub dependency
Supply chain riskContains a dependency which resolves to a GitHub URL. Dependencies fetched from GitHub specifiers are not immutable can be used to inject untrusted code or reduce the likelihood of a reproducible install.
Found 1 instance in 1 package
1396395
1.25%