diff --git a/native/Cargo.lock b/native/Cargo.lock index d297ecc07..be0931d5b 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -3,10 +3,17 @@ version = 4 [[package]] -name = "adler2" -version = "2.0.1" +name = "ahash" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "once_cell", + "version_check", + "zerocopy", +] [[package]] name = "aho-corasick" @@ -18,24 +25,10 @@ dependencies = [ ] [[package]] -name = "arboard" -version = "3.6.1" +name = "allocator-api2" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0348a1c054491f4bfe6ab86a7b6ab1e44e45d899005de92f58b3df180b36ddaf" -dependencies = [ - "clipboard-win", - "image", - "log", - "objc2", - "objc2-app-kit", - "objc2-core-foundation", - "objc2-core-graphics", - "objc2-foundation", - "parking_lot", - "percent-encoding", - "windows-sys 0.60.2", - "x11rb", -] +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "ast-grep-core" @@ -50,19 +43,19 @@ dependencies = [ ] [[package]] -name = "autocfg" -version = "1.5.0" +name = "astral-tl" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "d90933ffb0f97e2fc2e0de21da9d3f20597b804012d199843a6fe7c2810d28f3" +dependencies = [ + "memchr", +] [[package]] -name = "bincode" -version = "1.3.3" +name = "base64" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bit-set" @@ -96,18 +89,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bytemuck" -version = "1.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" - -[[package]] -name = "byteorder-lite" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" - [[package]] name = "cc" version = "1.2.56" @@ -125,12 +106,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] -name = "clipboard-win" -version = "5.4.1" +name = "const-random" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bde03770d3df201d4fb868f2c9c59e66a3e4e2bd06692a0fe701e7103c7e84d4" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" dependencies = [ - "error-code", + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom", + "once_cell", + "tiny-keccak", ] [[package]] @@ -142,15 +134,6 @@ dependencies = [ "unicode-segmentation", ] -[[package]] -name = "crc32fast" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" -dependencies = [ - "cfg-if", -] - [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -192,30 +175,6 @@ dependencies = [ "syn", ] -[[package]] -name = "dashmap" -version = "6.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" -dependencies = [ - "cfg-if", - "crossbeam-utils", - "hashbrown 0.14.5", - "lock_api", - "once_cell", - "parking_lot_core", -] - -[[package]] -name = "dispatch2" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" -dependencies = [ - "bitflags", - "objc2", -] - [[package]] name = "either" version = "1.15.0" @@ -246,68 +205,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" -[[package]] -name = "errno" -version = "0.3.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" -dependencies = [ - "libc", - "windows-sys 0.61.2", -] - -[[package]] -name = "error-code" -version = "3.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" - -[[package]] -name = "fancy-regex" -version = "0.16.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "998b056554fbe42e03ae0e152895cd1a7e1002aec800fdc6635d20270260c46f" -dependencies = [ - "bit-set", - "regex-automata", - "regex-syntax", -] - [[package]] name = "fastrand" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" -[[package]] -name = "fax" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f05de7d48f37cd6730705cbca900770cab77a89f413d23e100ad7fad7795a0ab" -dependencies = [ - "fax_derive", -] - -[[package]] -name = "fax_derive" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0aca10fb742cb43f9e7bb8467c91aa9bcb8e3ffbc6a6f7389bb93ffc920577d" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "fdeflate" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" -dependencies = [ - "simd-adler32", -] - [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -315,29 +218,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] -name = "flate2" -version = "1.1.9" +name = "foldhash" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" -dependencies = [ - "crc32fast", - "miniz_oxide", -] +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] -name = "fnv" -version = "1.0.7" +name = "getrandom" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "gethostname" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ - "rustix", - "windows-link", + "cfg-if", + "libc", + "wasi", ] [[package]] @@ -444,18 +338,11 @@ dependencies = [ name = "gsd-engine" version = "0.1.0" dependencies = [ - "arboard", - "dashmap", - "globset", - "gsd-ast", "gsd-grep", - "ignore", - "image", - "libc", + "html-to-markdown-rs", "napi", "napi-build", "napi-derive", - "syntect", ] [[package]] @@ -469,28 +356,52 @@ dependencies = [ "rayon", ] -[[package]] -name = "half" -version = "2.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" -dependencies = [ - "cfg-if", - "crunchy", - "zerocopy", -] - -[[package]] -name = "hashbrown" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" - [[package]] name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "html-escape" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d1ad449764d627e22bfd7cd5e8868264fc9236e07c752972b4080cd351cb476" +dependencies = [ + "utf8-width", +] + +[[package]] +name = "html-to-markdown-rs" +version = "2.28.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9377e16af590b764fd98fd176027cf8831c5335f8964f3f643753e38913a4e" +dependencies = [ + "ahash", + "astral-tl", + "base64", + "html-escape", + "html5ever", + "lru", + "once_cell", + "regex", + "thiserror", +] + +[[package]] +name = "html5ever" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1054432bae2f14e0061e33d23402fbaa67a921d319d56adc6bcf887ddad1cbc2" +dependencies = [ + "log", + "markup5ever", +] [[package]] name = "ignore" @@ -508,20 +419,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "image" -version = "0.25.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" -dependencies = [ - "bytemuck", - "byteorder-lite", - "moxcms", - "num-traits", - "png", - "tiff", -] - [[package]] name = "indexmap" version = "2.13.0" @@ -529,7 +426,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown", ] [[package]] @@ -554,12 +451,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "linux-raw-sys" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" - [[package]] name = "lock_api" version = "0.4.14" @@ -575,6 +466,26 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "lru" +version = "0.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "markup5ever" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8983d30f2915feeaaab2d6babdd6bc7e9ed1a00b66b5e6d74df19aa9c0e91862" +dependencies = [ + "log", + "tendril", + "web_atoms", +] + [[package]] name = "memchr" version = "2.8.0" @@ -590,26 +501,6 @@ dependencies = [ "libc", ] -[[package]] -name = "miniz_oxide" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" -dependencies = [ - "adler2", - "simd-adler32", -] - -[[package]] -name = "moxcms" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" -dependencies = [ - "num-traits", - "pxfm", -] - [[package]] name = "napi" version = "2.16.17" @@ -668,86 +559,10 @@ dependencies = [ ] [[package]] -name = "num-traits" -version = "0.2.19" +name = "new_debug_unreachable" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", -] - -[[package]] -name = "objc2" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" -dependencies = [ - "objc2-encode", -] - -[[package]] -name = "objc2-app-kit" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d49e936b501e5c5bf01fda3a9452ff86dc3ea98ad5f283e1455153142d97518c" -dependencies = [ - "bitflags", - "objc2", - "objc2-core-graphics", - "objc2-foundation", -] - -[[package]] -name = "objc2-core-foundation" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" -dependencies = [ - "bitflags", - "dispatch2", - "objc2", -] - -[[package]] -name = "objc2-core-graphics" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e022c9d066895efa1345f8e33e584b9f958da2fd4cd116792e15e07e4720a807" -dependencies = [ - "bitflags", - "dispatch2", - "objc2", - "objc2-core-foundation", - "objc2-io-surface", -] - -[[package]] -name = "objc2-encode" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" - -[[package]] -name = "objc2-foundation" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" -dependencies = [ - "bitflags", - "objc2", - "objc2-core-foundation", -] - -[[package]] -name = "objc2-io-surface" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180788110936d59bab6bd83b6060ffdfffb3b922ba1396b312ae795e1de9d81d" -dependencies = [ - "bitflags", - "objc2", - "objc2-core-foundation", -] +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" [[package]] name = "once_cell" @@ -778,12 +593,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "percent-encoding" -version = "2.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" - [[package]] name = "phf" version = "0.13.1" @@ -795,6 +604,16 @@ dependencies = [ "serde", ] +[[package]] +name = "phf_codegen" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1" +dependencies = [ + "phf_generator", + "phf_shared", +] + [[package]] name = "phf_generator" version = "0.13.1" @@ -828,17 +647,10 @@ dependencies = [ ] [[package]] -name = "png" -version = "0.18.1" +name = "precomputed-hash" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" -dependencies = [ - "bitflags", - "crc32fast", - "fdeflate", - "flate2", - "miniz_oxide", -] +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" [[package]] name = "proc-macro2" @@ -849,18 +661,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "pxfm" -version = "0.1.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a041e753da8b807c9255f28de81879c78c876392ff2469cde94799b2896b9d" - -[[package]] -name = "quick-error" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" - [[package]] name = "quote" version = "1.0.45" @@ -928,19 +728,6 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" -[[package]] -name = "rustix" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys", - "windows-sys 0.61.2", -] - [[package]] name = "same-file" version = "1.0.6" @@ -1011,12 +798,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" -[[package]] -name = "simd-adler32" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" - [[package]] name = "siphasher" version = "1.0.2" @@ -1035,6 +816,30 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" +[[package]] +name = "string_cache" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a18596f8c785a729f2819c0f6a7eae6ebeebdfffbfe4214ae6b087f690e31901" +dependencies = [ + "new_debug_unreachable", + "parking_lot", + "phf_shared", + "precomputed-hash", +] + +[[package]] +name = "string_cache_codegen" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "585635e46db231059f76c5849798146164652513eb9e8ab2685939dd90f29b69" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", +] + [[package]] name = "syn" version = "2.0.117" @@ -1047,21 +852,13 @@ dependencies = [ ] [[package]] -name = "syntect" -version = "5.3.0" +name = "tendril" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "656b45c05d95a5704399aeef6bd0ddec7b2b3531b7c9e900abbf7c4d2190c925" +checksum = "c4790fc369d5a530f4b544b094e31388b9b3a37c0f4652ade4505945f5660d24" dependencies = [ - "bincode", - "fancy-regex", - "flate2", - "fnv", - "once_cell", - "regex-syntax", - "serde", - "serde_derive", - "thiserror", - "walkdir", + "new_debug_unreachable", + "utf-8", ] [[package]] @@ -1085,17 +882,12 @@ dependencies = [ ] [[package]] -name = "tiff" -version = "0.11.3" +name = "tiny-keccak" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b63feaf3343d35b6ca4d50483f94843803b0f51634937cc2ec519fc32232bc52" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" dependencies = [ - "fax", - "flate2", - "half", - "quick-error", - "weezl", - "zune-jpeg", + "crunchy", ] [[package]] @@ -1500,6 +1292,24 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1292c0d970b54115d14f2492fe0170adf21d68a1de108eebc51c1df4f346a091" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "walkdir" version = "2.5.0" @@ -1511,10 +1321,22 @@ dependencies = [ ] [[package]] -name = "weezl" -version = "0.1.12" +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "web_atoms" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a9779e9f04d2ac1ce317aee707aa2f6b773afba7b931222bff6983843b1576" +dependencies = [ + "phf", + "phf_codegen", + "string_cache", + "string_cache_codegen", +] [[package]] name = "winapi-util" @@ -1522,7 +1344,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -1531,15 +1353,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -1549,88 +1362,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - -[[package]] -name = "x11rb" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9993aa5be5a26815fe2c3eacfc1fde061fc1a1f094bf1ad2a18bf9c495dd7414" -dependencies = [ - "gethostname", - "rustix", - "x11rb-protocol", -] - -[[package]] -name = "x11rb-protocol" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd" - [[package]] name = "zerocopy" version = "0.8.42" @@ -1656,18 +1387,3 @@ name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" - -[[package]] -name = "zune-core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9" - -[[package]] -name = "zune-jpeg" -version = "0.5.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec5f41c76397b7da451efd19915684f727d7e1d516384ca6bd0ec43ec94de23c" -dependencies = [ - "zune-core", -] diff --git a/native/crates/engine/Cargo.toml b/native/crates/engine/Cargo.toml index 82917751e..cc4c4edb1 100644 --- a/native/crates/engine/Cargo.toml +++ b/native/crates/engine/Cargo.toml @@ -16,6 +16,7 @@ gsd-grep = { path = "../grep" } arboard = "3" dashmap = "6" globset = "0.4" +html-to-markdown-rs = { version = "2", default-features = false } ignore = "0.4" image = { version = "0.25", default-features = false, features = ["png"] } napi = { version = "2", features = ["napi8"] } diff --git a/native/crates/engine/src/html.rs b/native/crates/engine/src/html.rs new file mode 100644 index 000000000..2cc44c047 --- /dev/null +++ b/native/crates/engine/src/html.rs @@ -0,0 +1,44 @@ +//! HTML to Markdown conversion via N-API. +//! +//! Wraps `html-to-markdown-rs` and exposes it as a JS-callable N-API export. + +use html_to_markdown_rs::{convert, ConversionOptions, PreprocessingOptions, PreprocessingPreset}; +use napi::bindgen_prelude::*; +use napi_derive::napi; + +/// Options for HTML to Markdown conversion. +#[napi(object)] +#[derive(Debug, Default)] +pub struct HtmlToMarkdownOptions { + /// Remove navigation elements, forms, headers, footers. + #[napi(js_name = "cleanContent")] + pub clean_content: Option, + /// Skip images during conversion. + #[napi(js_name = "skipImages")] + pub skip_images: Option, +} + +/// Convert HTML source to Markdown with optional preprocessing. +/// +/// Strips boilerplate (nav, forms, headers, footers) when `cleanContent` is true. +/// Returns the Markdown string. +#[napi(js_name = "htmlToMarkdown")] +pub fn html_to_markdown(html: String, options: Option) -> Result { + let options = options.unwrap_or_default(); + let clean_content = options.clean_content.unwrap_or(false); + let skip_images = options.skip_images.unwrap_or(false); + + let conversion_opts = ConversionOptions { + skip_images, + preprocessing: PreprocessingOptions { + enabled: clean_content, + preset: PreprocessingPreset::Aggressive, + remove_navigation: true, + remove_forms: true, + }, + ..Default::default() + }; + + convert(&html, Some(conversion_opts)) + .map_err(|err| Error::from_reason(format!("HTML conversion error: {err}"))) +} diff --git a/native/crates/engine/src/lib.rs b/native/crates/engine/src/lib.rs index 012a80bd4..803c7112f 100644 --- a/native/crates/engine/src/lib.rs +++ b/native/crates/engine/src/lib.rs @@ -15,5 +15,6 @@ mod glob_util; mod ast; mod grep; mod highlight; +mod html; mod ps; mod task; diff --git a/packages/native/package.json b/packages/native/package.json index adcfb2e48..4a139b54d 100644 --- a/packages/native/package.json +++ b/packages/native/package.json @@ -8,7 +8,7 @@ "scripts": { "build:native": "node ../../native/scripts/build.js", "build:native:dev": "node ../../native/scripts/build.js --dev", - "test": "node --test src/__tests__/grep.test.mjs src/__tests__/ps.test.mjs src/__tests__/glob.test.mjs src/__tests__/clipboard.test.mjs src/__tests__/highlight.test.mjs" + "test": "node --test src/__tests__/grep.test.mjs src/__tests__/ps.test.mjs src/__tests__/glob.test.mjs src/__tests__/clipboard.test.mjs src/__tests__/highlight.test.mjs src/__tests__/html.test.mjs" }, "exports": { ".": { @@ -34,6 +34,10 @@ "./ast": { "types": "./src/ast/index.ts", "import": "./src/ast/index.ts" + }, + "./html": { + "types": "./src/html/index.ts", + "import": "./src/html/index.ts" } }, "files": [ diff --git a/packages/native/src/__tests__/html.test.mjs b/packages/native/src/__tests__/html.test.mjs new file mode 100644 index 000000000..31e21c463 --- /dev/null +++ b/packages/native/src/__tests__/html.test.mjs @@ -0,0 +1,98 @@ +import { test, describe } from "node:test"; +import assert from "node:assert/strict"; +import { createRequire } from "node:module"; +import * as path from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const require = createRequire(import.meta.url); + +const addonDir = path.resolve(__dirname, "..", "..", "..", "..", "native", "addon"); +const platformTag = `${process.platform}-${process.arch}`; +const candidates = [ + path.join(addonDir, `gsd_engine.${platformTag}.node`), + path.join(addonDir, "gsd_engine.dev.node"), +]; + +let native; +for (const candidate of candidates) { + try { + native = require(candidate); + break; + } catch { + // try next + } +} + +if (!native) { + console.error("Native addon not found. Run `npm run build:native -w @gsd/native` first."); + process.exit(1); +} + +describe("native html: htmlToMarkdown()", () => { + test("converts basic HTML to markdown", () => { + const html = "

Hello

World

"; + const result = native.htmlToMarkdown(html); + assert.ok(result.includes("Hello"), "Should contain heading text"); + assert.ok(result.includes("World"), "Should contain paragraph text"); + }); + + test("converts links to markdown links", () => { + const html = '

Visit Example

'; + const result = native.htmlToMarkdown(html); + assert.ok(result.includes("[Example]"), "Should contain markdown link text"); + assert.ok(result.includes("(https://example.com)"), "Should contain markdown link URL"); + }); + + test("converts lists to markdown", () => { + const html = "
  • First
  • Second
  • Third
"; + const result = native.htmlToMarkdown(html); + assert.ok(result.includes("First"), "Should contain first item"); + assert.ok(result.includes("Second"), "Should contain second item"); + assert.ok(result.includes("Third"), "Should contain third item"); + }); + + test("converts bold and italic", () => { + const html = "

bold and italic

"; + const result = native.htmlToMarkdown(html); + assert.ok(result.includes("**bold**") || result.includes("__bold__"), "Should contain bold"); + assert.ok(result.includes("*italic*") || result.includes("_italic_"), "Should contain italic"); + }); + + test("handles empty HTML", () => { + const result = native.htmlToMarkdown(""); + assert.equal(typeof result, "string"); + }); + + test("handles plain text", () => { + const result = native.htmlToMarkdown("Just plain text"); + assert.ok(result.includes("Just plain text"), "Should preserve plain text"); + }); + + test("accepts skipImages option", () => { + const html = '

Title

Content with photo image

'; + const result = native.htmlToMarkdown(html, { skipImages: true }); + assert.ok(result.includes("Title"), "Should contain heading"); + assert.ok(result.includes("Content"), "Should contain paragraph text"); + }); + + test("accepts cleanContent option", () => { + const html = '

Article

Body text.

Copyright
'; + const result = native.htmlToMarkdown(html, { cleanContent: true }); + assert.ok(result.includes("Article") || result.includes("Body text"), "Should contain main content"); + }); + + test("converts code blocks", () => { + const html = "
const x = 1;
"; + const result = native.htmlToMarkdown(html); + assert.ok(result.includes("const x = 1;"), "Should contain code content"); + }); + + test("converts complex nested HTML", () => { + const html = '

Section

Text with bold link.

  • Item one
  • Item two
'; + const result = native.htmlToMarkdown(html); + assert.ok(result.includes("Section"), "Should contain heading"); + assert.ok(result.includes("example.com"), "Should contain link"); + assert.ok(result.includes("one"), "Should contain list items"); + }); +}); diff --git a/packages/native/src/html/index.ts b/packages/native/src/html/index.ts new file mode 100644 index 000000000..28886b7a2 --- /dev/null +++ b/packages/native/src/html/index.ts @@ -0,0 +1,24 @@ +/** + * HTML to Markdown conversion via native Rust bindings. + * + * Uses `html-to-markdown-rs` under the hood for high-performance + * conversion with optional content cleaning (stripping nav, forms, etc.). + */ + +import { native } from "../native.js"; +import type { HtmlToMarkdownOptions } from "./types.js"; + +export type { HtmlToMarkdownOptions }; + +/** + * Convert an HTML string to Markdown. + * + * When `cleanContent` is true, boilerplate elements (nav, forms, headers, + * footers) are stripped before conversion. + */ +export function htmlToMarkdown( + html: string, + options?: HtmlToMarkdownOptions, +): string { + return native.htmlToMarkdown(html, options ?? {}) as string; +} diff --git a/packages/native/src/html/types.ts b/packages/native/src/html/types.ts new file mode 100644 index 000000000..a8984c7a8 --- /dev/null +++ b/packages/native/src/html/types.ts @@ -0,0 +1,7 @@ +/** Options for HTML to Markdown conversion. */ +export interface HtmlToMarkdownOptions { + /** Remove navigation elements, forms, headers, footers. */ + cleanContent?: boolean; + /** Skip images during conversion. */ + skipImages?: boolean; +} diff --git a/packages/native/src/index.ts b/packages/native/src/index.ts index c43f05ab8..4134f350c 100644 --- a/packages/native/src/index.ts +++ b/packages/native/src/index.ts @@ -7,6 +7,7 @@ * - ps: cross-platform process tree management * - glob: gitignore-respecting filesystem discovery with scan caching * - highlight: syntect-based syntax highlighting + * - html: HTML to Markdown conversion */ export { @@ -54,3 +55,6 @@ export type { AstFindMatch, AstFindOptions, AstFindResult, AstReplaceChange, AstReplaceFileChange, AstReplaceOptions, AstReplaceResult, } from "./ast/index.js"; + +export { htmlToMarkdown } from "./html/index.js"; +export type { HtmlToMarkdownOptions } from "./html/index.js"; diff --git a/packages/native/src/native.ts b/packages/native/src/native.ts index 039efc10e..729088c30 100644 --- a/packages/native/src/native.ts +++ b/packages/native/src/native.ts @@ -60,4 +60,5 @@ export const native = loadNative() as { readImageFromClipboard: () => Promise; astGrep: (options: unknown) => unknown; astEdit: (options: unknown) => unknown; + htmlToMarkdown: (html: string, options: unknown) => unknown; };