Astro+Directus(Visual-Editing)+Snipcart Test-Stack
This commit is contained in:
till committed
commit
1459e99cb0
10119 files changed
+1499318
No files matched your search
+11
@@ -0,0 +1,11 @@
|
||||
export const affixSymbol: RegExp;
|
||||
export const newLine: RegExp;
|
||||
export const terminalMarker: RegExp;
|
||||
export const wordSymbolInner: RegExp;
|
||||
export const numerical: RegExp;
|
||||
export const digitStart: RegExp;
|
||||
export const lowerInitial: RegExp;
|
||||
export const surrogates: RegExp;
|
||||
export const punctuation: RegExp;
|
||||
export const word: RegExp;
|
||||
export const whiteSpace: RegExp;
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
// This module is generated by `script/build-expressions.js`.
|
||||
export const affixSymbol =
|
||||
/^([!"').?\u0F3B\u0F3D\u169C\u2019\u201D\u2026\u203A\u203D\u2046\u207E\u208E\u2309\u230B\u232A\u2769\u276B\u276D\u276F\u2771\u2773\u2775\u27C6\u27E7\u27E9\u27EB\u27ED\u27EF\u2984\u2986\u2988\u298A\u298C\u298E\u2990\u2992\u2994\u2996\u2998\u29D9\u29DB\u29FD\u2E03\u2E05\u2E0A\u2E0D\u2E1D\u2E21\u2E23\u2E25\u2E27\u2E29\u2E56\u2E58\u2E5A\u2E5C\u3009\u300B\u300D\u300F\u3011\u3015\u3017\u3019\u301B\u301E\u301F\uFD3E\uFE18\uFE36\uFE38\uFE3A\uFE3C\uFE3E\uFE40\uFE42\uFE44\uFE48\uFE5A\uFE5C\uFE5E\uFF09\uFF3D\uFF5D\uFF60\uFF63\u00BB\]}])\1*$/
|
||||
export const newLine = /^[ \t]*((\r?\n|\r)[\t ]*)+$/
|
||||
export const terminalMarker = /^([!.?\u2026\u203D]+)$/
|
||||
export const wordSymbolInner =
|
||||
/^([&'\-.:=?@\u00AD\u00B7\u2010\u2011\u2019\u2027]|_+)$/
|
||||
export const numerical =
|
||||
/^(?:[\d\u00B2\u00B3\u00B9\u00BC-\u00BE\u0660-\u0669\u06F0-\u06F9\u07C0-\u07C9\u0966-\u096F\u09E6-\u09EF\u09F4-\u09F9\u0A66-\u0A6F\u0AE6-\u0AEF\u0B66-\u0B6F\u0B72-\u0B77\u0BE6-\u0BF2\u0C66-\u0C6F\u0C78-\u0C7E\u0CE6-\u0CEF\u0D58-\u0D5E\u0D66-\u0D78\u0DE6-\u0DEF\u0E50-\u0E59\u0ED0-\u0ED9\u0F20-\u0F33\u1040-\u1049\u1090-\u1099\u1369-\u137C\u16EE-\u16F0\u17E0-\u17E9\u17F0-\u17F9\u1810-\u1819\u1946-\u194F\u19D0-\u19DA\u1A80-\u1A89\u1A90-\u1A99\u1B50-\u1B59\u1BB0-\u1BB9\u1C40-\u1C49\u1C50-\u1C59\u2070\u2074-\u2079\u2080-\u2089\u2150-\u2182\u2185-\u2189\u2460-\u249B\u24EA-\u24FF\u2776-\u2793\u2CFD\u3007\u3021-\u3029\u3038-\u303A\u3192-\u3195\u3220-\u3229\u3248-\u324F\u3251-\u325F\u3280-\u3289\u32B1-\u32BF\uA620-\uA629\uA6E6-\uA6EF\uA830-\uA835\uA8D0-\uA8D9\uA900-\uA909\uA9D0-\uA9D9\uA9F0-\uA9F9\uAA50-\uAA59\uABF0-\uABF9\uFF10-\uFF19]|\uD800[\uDD07-\uDD33\uDD40-\uDD78\uDD8A\uDD8B\uDEE1-\uDEFB\uDF20-\uDF23\uDF41\uDF4A\uDFD1-\uDFD5]|\uD801[\uDCA0-\uDCA9]|\uD802[\uDC58-\uDC5F\uDC79-\uDC7F\uDCA7-\uDCAF\uDCFB-\uDCFF\uDD16-\uDD1B\uDDBC\uDDBD\uDDC0-\uDDCF\uDDD2-\uDDFF\uDE40-\uDE48\uDE7D\uDE7E\uDE9D-\uDE9F\uDEEB-\uDEEF\uDF58-\uDF5F\uDF78-\uDF7F\uDFA9-\uDFAF]|\uD803[\uDCFA-\uDCFF\uDD30-\uDD39\uDE60-\uDE7E\uDF1D-\uDF26\uDF51-\uDF54\uDFC5-\uDFCB]|\uD804[\uDC52-\uDC6F\uDCF0-\uDCF9\uDD36-\uDD3F\uDDD0-\uDDD9\uDDE1-\uDDF4\uDEF0-\uDEF9]|\uD805[\uDC50-\uDC59\uDCD0-\uDCD9\uDE50-\uDE59\uDEC0-\uDEC9\uDF30-\uDF3B]|\uD806[\uDCE0-\uDCF2\uDD50-\uDD59]|\uD807[\uDC50-\uDC6C\uDD50-\uDD59\uDDA0-\uDDA9\uDF50-\uDF59\uDFC0-\uDFD4]|\uD809[\uDC00-\uDC6E]|\uD81A[\uDE60-\uDE69\uDEC0-\uDEC9\uDF50-\uDF59\uDF5B-\uDF61]|\uD81B[\uDE80-\uDE96]|\uD834[\uDEC0-\uDED3\uDEE0-\uDEF3\uDF60-\uDF78]|\uD835[\uDFCE-\uDFFF]|\uD838[\uDD40-\uDD49\uDEF0-\uDEF9]|\uD839[\uDCF0-\uDCF9]|\uD83A[\uDCC7-\uDCCF\uDD50-\uDD59]|\uD83B[\uDC71-\uDCAB\uDCAD-\uDCAF\uDCB1-\uDCB4\uDD01-\uDD2D\uDD2F-\uDD3D]|\uD83C[\uDD00-\uDD0C]|\uD83E[\uDFF0-\uDFF9])+$/
|
||||
export const digitStart = /^\d/
|
||||
export const lowerInitial =
|
||||
/^(?:[a-z\u00B5\u00DF-\u00F6\u00F8-\u00FF\u0101\u0103\u0105\u0107\u0109\u010B\u010D\u010F\u0111\u0113\u0115\u0117\u0119\u011B\u011D\u011F\u0121\u0123\u0125\u0127\u0129\u012B\u012D\u012F\u0131\u0133\u0135\u0137\u0138\u013A\u013C\u013E\u0140\u0142\u0144\u0146\u0148\u0149\u014B\u014D\u014F\u0151\u0153\u0155\u0157\u0159\u015B\u015D\u015F\u0161\u0163\u0165\u0167\u0169\u016B\u016D\u016F\u0171\u0173\u0175\u0177\u017A\u017C\u017E-\u0180\u0183\u0185\u0188\u018C\u018D\u0192\u0195\u0199-\u019B\u019E\u01A1\u01A3\u01A5\u01A8\u01AA\u01AB\u01AD\u01B0\u01B4\u01B6\u01B9\u01BA\u01BD-\u01BF\u01C6\u01C9\u01CC\u01CE\u01D0\u01D2\u01D4\u01D6\u01D8\u01DA\u01DC\u01DD\u01DF\u01E1\u01E3\u01E5\u01E7\u01E9\u01EB\u01ED\u01EF\u01F0\u01F3\u01F5\u01F9\u01FB\u01FD\u01FF\u0201\u0203\u0205\u0207\u0209\u020B\u020D\u020F\u0211\u0213\u0215\u0217\u0219\u021B\u021D\u021F\u0221\u0223\u0225\u0227\u0229\u022B\u022D\u022F\u0231\u0233-\u0239\u023C\u023F\u0240\u0242\u0247\u0249\u024B\u024D\u024F-\u0293\u0295-\u02AF\u0371\u0373\u0377\u037B-\u037D\u0390\u03AC-\u03CE\u03D0\u03D1\u03D5-\u03D7\u03D9\u03DB\u03DD\u03DF\u03E1\u03E3\u03E5\u03E7\u03E9\u03EB\u03ED\u03EF-\u03F3\u03F5\u03F8\u03FB\u03FC\u0430-\u045F\u0461\u0463\u0465\u0467\u0469\u046B\u046D\u046F\u0471\u0473\u0475\u0477\u0479\u047B\u047D\u047F\u0481\u048B\u048D\u048F\u0491\u0493\u0495\u0497\u0499\u049B\u049D\u049F\u04A1\u04A3\u04A5\u04A7\u04A9\u04AB\u04AD\u04AF\u04B1\u04B3\u04B5\u04B7\u04B9\u04BB\u04BD\u04BF\u04C2\u04C4\u04C6\u04C8\u04CA\u04CC\u04CE\u04CF\u04D1\u04D3\u04D5\u04D7\u04D9\u04DB\u04DD\u04DF\u04E1\u04E3\u04E5\u04E7\u04E9\u04EB\u04ED\u04EF\u04F1\u04F3\u04F5\u04F7\u04F9\u04FB\u04FD\u04FF\u0501\u0503\u0505\u0507\u0509\u050B\u050D\u050F\u0511\u0513\u0515\u0517\u0519\u051B\u051D\u051F\u0521\u0523\u0525\u0527\u0529\u052B\u052D\u052F\u0560-\u0588\u10D0-\u10FA\u10FD-\u10FF\u13F8-\u13FD\u1C80-\u1C88\u1D00-\u1D2B\u1D6B-\u1D77\u1D79-\u1D9A\u1E01\u1E03\u1E05\u1E07\u1E09\u1E0B\u1E0D\u1E0F\u1E11\u1E13\u1E15\u1E17\u1E19\u1E1B\u1E1D\u1E1F\u1E21\u1E23\u1E25\u1E27\u1E29\u1E2B\u1E2D\u1E2F\u1E31\u1E33\u1E35\u1E37\u1E39\u1E3B\u1E3D\u1E3F\u1E41\u1E43\u1E45\u1E47\u1E49\u1E4B\u1E4D\u1E4F\u1E51\u1E53\u1E55\u1E57\u1E59\u1E5B\u1E5D\u1E5F\u1E61\u1E63\u1E65\u1E67\u1E69\u1E6B\u1E6D\u1E6F\u1E71\u1E73\u1E75\u1E77\u1E79\u1E7B\u1E7D\u1E7F\u1E81\u1E83\u1E85\u1E87\u1E89\u1E8B\u1E8D\u1E8F\u1E91\u1E93\u1E95-\u1E9D\u1E9F\u1EA1\u1EA3\u1EA5\u1EA7\u1EA9\u1EAB\u1EAD\u1EAF\u1EB1\u1EB3\u1EB5\u1EB7\u1EB9\u1EBB\u1EBD\u1EBF\u1EC1\u1EC3\u1EC5\u1EC7\u1EC9\u1ECB\u1ECD\u1ECF\u1ED1\u1ED3\u1ED5\u1ED7\u1ED9\u1EDB\u1EDD\u1EDF\u1EE1\u1EE3\u1EE5\u1EE7\u1EE9\u1EEB\u1EED\u1EEF\u1EF1\u1EF3\u1EF5\u1EF7\u1EF9\u1EFB\u1EFD\u1EFF-\u1F07\u1F10-\u1F15\u1F20-\u1F27\u1F30-\u1F37\u1F40-\u1F45\u1F50-\u1F57\u1F60-\u1F67\u1F70-\u1F7D\u1F80-\u1F87\u1F90-\u1F97\u1FA0-\u1FA7\u1FB0-\u1FB4\u1FB6\u1FB7\u1FBE\u1FC2-\u1FC4\u1FC6\u1FC7\u1FD0-\u1FD3\u1FD6\u1FD7\u1FE0-\u1FE7\u1FF2-\u1FF4\u1FF6\u1FF7\u210A\u210E\u210F\u2113\u212F\u2134\u2139\u213C\u213D\u2146-\u2149\u214E\u2184\u2C30-\u2C5F\u2C61\u2C65\u2C66\u2C68\u2C6A\u2C6C\u2C71\u2C73\u2C74\u2C76-\u2C7B\u2C81\u2C83\u2C85\u2C87\u2C89\u2C8B\u2C8D\u2C8F\u2C91\u2C93\u2C95\u2C97\u2C99\u2C9B\u2C9D\u2C9F\u2CA1\u2CA3\u2CA5\u2CA7\u2CA9\u2CAB\u2CAD\u2CAF\u2CB1\u2CB3\u2CB5\u2CB7\u2CB9\u2CBB\u2CBD\u2CBF\u2CC1\u2CC3\u2CC5\u2CC7\u2CC9\u2CCB\u2CCD\u2CCF\u2CD1\u2CD3\u2CD5\u2CD7\u2CD9\u2CDB\u2CDD\u2CDF\u2CE1\u2CE3\u2CE4\u2CEC\u2CEE\u2CF3\u2D00-\u2D25\u2D27\u2D2D\uA641\uA643\uA645\uA647\uA649\uA64B\uA64D\uA64F\uA651\uA653\uA655\uA657\uA659\uA65B\uA65D\uA65F\uA661\uA663\uA665\uA667\uA669\uA66B\uA66D\uA681\uA683\uA685\uA687\uA689\uA68B\uA68D\uA68F\uA691\uA693\uA695\uA697\uA699\uA69B\uA723\uA725\uA727\uA729\uA72B\uA72D\uA72F-\uA731\uA733\uA735\uA737\uA739\uA73B\uA73D\uA73F\uA741\uA743\uA745\uA747\uA749\uA74B\uA74D\uA74F\uA751\uA753\uA755\uA757\uA759\uA75B\uA75D\uA75F\uA761\uA763\uA765\uA767\uA769\uA76B\uA76D\uA76F\uA771-\uA778\uA77A\uA77C\uA77F\uA781\uA783\uA785\uA787\uA78C\uA78E\uA791\uA793-\uA795\uA797\uA799\uA79B\uA79D\uA79F\uA7A1\uA7A3\uA7A5\uA7A7\uA7A9\uA7AF\uA7B5\uA7B7\uA7B9\uA7BB\uA7BD\uA7BF\uA7C1\uA7C3\uA7C8\uA7CA\uA7D1\uA7D3\uA7D5\uA7D7\uA7D9\uA7F6\uA7FA\uAB30-\uAB5A\uAB60-\uAB68\uAB70-\uABBF\uFB00-\uFB06\uFB13-\uFB17\uFF41-\uFF5A]|\uD801[\uDC28-\uDC4F\uDCD8-\uDCFB\uDD97-\uDDA1\uDDA3-\uDDB1\uDDB3-\uDDB9\uDDBB\uDDBC]|\uD803[\uDCC0-\uDCF2]|\uD806[\uDCC0-\uDCDF]|\uD81B[\uDE60-\uDE7F]|\uD835[\uDC1A-\uDC33\uDC4E-\uDC54\uDC56-\uDC67\uDC82-\uDC9B\uDCB6-\uDCB9\uDCBB\uDCBD-\uDCC3\uDCC5-\uDCCF\uDCEA-\uDD03\uDD1E-\uDD37\uDD52-\uDD6B\uDD86-\uDD9F\uDDBA-\uDDD3\uDDEE-\uDE07\uDE22-\uDE3B\uDE56-\uDE6F\uDE8A-\uDEA5\uDEC2-\uDEDA\uDEDC-\uDEE1\uDEFC-\uDF14\uDF16-\uDF1B\uDF36-\uDF4E\uDF50-\uDF55\uDF70-\uDF88\uDF8A-\uDF8F\uDFAA-\uDFC2\uDFC4-\uDFC9\uDFCB]|\uD837[\uDF00-\uDF09\uDF0B-\uDF1E\uDF25-\uDF2A]|\uD83A[\uDD22-\uDD43])/
|
||||
export const surrogates = /[\uD800-\uDFFF]/
|
||||
export const punctuation =
|
||||
/[!"'-),-/:;?[-\]_{}\u00A1\u00A7\u00AB\u00B6\u00B7\u00BB\u00BF\u037E\u0387\u055A-\u055F\u0589\u058A\u05BE\u05C0\u05C3\u05C6\u05F3\u05F4\u0609\u060A\u060C\u060D\u061B\u061D-\u061F\u066A-\u066D\u06D4\u0700-\u070D\u07F7-\u07F9\u0830-\u083E\u085E\u0964\u0965\u0970\u09FD\u0A76\u0AF0\u0C77\u0C84\u0DF4\u0E4F\u0E5A\u0E5B\u0F04-\u0F12\u0F14\u0F3A-\u0F3D\u0F85\u0FD0-\u0FD4\u0FD9\u0FDA\u104A-\u104F\u10FB\u1360-\u1368\u1400\u166E\u169B\u169C\u16EB-\u16ED\u1735\u1736\u17D4-\u17D6\u17D8-\u17DA\u1800-\u180A\u1944\u1945\u1A1E\u1A1F\u1AA0-\u1AA6\u1AA8-\u1AAD\u1B5A-\u1B60\u1B7D\u1B7E\u1BFC-\u1BFF\u1C3B-\u1C3F\u1C7E\u1C7F\u1CC0-\u1CC7\u1CD3\u2010-\u201F\u2022-\u2027\u2032-\u203A\u203C-\u2043\u2045-\u2051\u2053-\u205E\u207D\u207E\u208D\u208E\u2308-\u230B\u2329\u232A\u2768-\u2775\u27C5\u27C6\u27E6-\u27EF\u2983-\u2998\u29D8-\u29DB\u29FC\u29FD\u2CF9-\u2CFC\u2CFE\u2CFF\u2D70\u2E00-\u2E2E\u2E30-\u2E4F\u2E52-\u2E5D\u3001-\u3003\u3008-\u3011\u3014-\u301F\u3030\u303D\u30A0\u30FB\uA4FE\uA4FF\uA60D-\uA60F\uA673\uA67E\uA6F2-\uA6F7\uA874-\uA877\uA8CE\uA8CF\uA8F8-\uA8FA\uA8FC\uA92E\uA92F\uA95F\uA9C1-\uA9CD\uA9DE\uA9DF\uAA5C-\uAA5F\uAADE\uAADF\uAAF0\uAAF1\uABEB\uFD3E\uFD3F\uFE10-\uFE19\uFE30-\uFE52\uFE54-\uFE61\uFE63\uFE68\uFE6A\uFE6B\uFF01-\uFF03\uFF05-\uFF0A\uFF0C-\uFF0F\uFF1A\uFF1B\uFF1F\uFF20\uFF3B-\uFF3D\uFF3F\uFF5B\uFF5D\uFF5F-\uFF65]|\uD800[\uDD00-\uDD02\uDF9F\uDFD0]|\uD801\uDD6F|\uD802[\uDC57\uDD1F\uDD3F\uDE50-\uDE58\uDE7F\uDEF0-\uDEF6\uDF39-\uDF3F\uDF99-\uDF9C]|\uD803[\uDEAD\uDF55-\uDF59\uDF86-\uDF89]|\uD804[\uDC47-\uDC4D\uDCBB\uDCBC\uDCBE-\uDCC1\uDD40-\uDD43\uDD74\uDD75\uDDC5-\uDDC8\uDDCD\uDDDB\uDDDD-\uDDDF\uDE38-\uDE3D\uDEA9]|\uD805[\uDC4B-\uDC4F\uDC5A\uDC5B\uDC5D\uDCC6\uDDC1-\uDDD7\uDE41-\uDE43\uDE60-\uDE6C\uDEB9\uDF3C-\uDF3E]|\uD806[\uDC3B\uDD44-\uDD46\uDDE2\uDE3F-\uDE46\uDE9A-\uDE9C\uDE9E-\uDEA2\uDF00-\uDF09]|\uD807[\uDC41-\uDC45\uDC70\uDC71\uDEF7\uDEF8\uDF43-\uDF4F\uDFFF]|\uD809[\uDC70-\uDC74]|\uD80B[\uDFF1\uDFF2]|\uD81A[\uDE6E\uDE6F\uDEF5\uDF37-\uDF3B\uDF44]|\uD81B[\uDE97-\uDE9A\uDFE2]|\uD82F\uDC9F|\uD836[\uDE87-\uDE8B]|\uD83A[\uDD5E\uDD5F]/
|
||||
export const word =
|
||||
/[\dA-Za-z\u00AA\u00B2\u00B3\u00B5\u00B9\u00BA\u00BC-\u00BE\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02C1\u02C6-\u02D1\u02E0-\u02E4\u02EC\u02EE\u0300-\u0374\u0376\u0377\u037A-\u037D\u037F\u0386\u0388-\u038A\u038C\u038E-\u03A1\u03A3-\u03F5\u03F7-\u0481\u0483-\u052F\u0531-\u0556\u0559\u0560-\u0588\u0591-\u05BD\u05BF\u05C1\u05C2\u05C4\u05C5\u05C7\u05D0-\u05EA\u05EF-\u05F2\u0610-\u061A\u0620-\u0669\u066E-\u06D3\u06D5-\u06DC\u06DF-\u06E8\u06EA-\u06FC\u06FF\u0710-\u074A\u074D-\u07B1\u07C0-\u07F5\u07FA\u07FD\u0800-\u082D\u0840-\u085B\u0860-\u086A\u0870-\u0887\u0889-\u088E\u0898-\u08E1\u08E3-\u0963\u0966-\u096F\u0971-\u0983\u0985-\u098C\u098F\u0990\u0993-\u09A8\u09AA-\u09B0\u09B2\u09B6-\u09B9\u09BC-\u09C4\u09C7\u09C8\u09CB-\u09CE\u09D7\u09DC\u09DD\u09DF-\u09E3\u09E6-\u09F1\u09F4-\u09F9\u09FC\u09FE\u0A01-\u0A03\u0A05-\u0A0A\u0A0F\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32\u0A33\u0A35\u0A36\u0A38\u0A39\u0A3C\u0A3E-\u0A42\u0A47\u0A48\u0A4B-\u0A4D\u0A51\u0A59-\u0A5C\u0A5E\u0A66-\u0A75\u0A81-\u0A83\u0A85-\u0A8D\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2\u0AB3\u0AB5-\u0AB9\u0ABC-\u0AC5\u0AC7-\u0AC9\u0ACB-\u0ACD\u0AD0\u0AE0-\u0AE3\u0AE6-\u0AEF\u0AF9-\u0AFF\u0B01-\u0B03\u0B05-\u0B0C\u0B0F\u0B10\u0B13-\u0B28\u0B2A-\u0B30\u0B32\u0B33\u0B35-\u0B39\u0B3C-\u0B44\u0B47\u0B48\u0B4B-\u0B4D\u0B55-\u0B57\u0B5C\u0B5D\u0B5F-\u0B63\u0B66-\u0B6F\u0B71-\u0B77\u0B82\u0B83\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99\u0B9A\u0B9C\u0B9E\u0B9F\u0BA3\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0BBE-\u0BC2\u0BC6-\u0BC8\u0BCA-\u0BCD\u0BD0\u0BD7\u0BE6-\u0BF2\u0C00-\u0C0C\u0C0E-\u0C10\u0C12-\u0C28\u0C2A-\u0C39\u0C3C-\u0C44\u0C46-\u0C48\u0C4A-\u0C4D\u0C55\u0C56\u0C58-\u0C5A\u0C5D\u0C60-\u0C63\u0C66-\u0C6F\u0C78-\u0C7E\u0C80-\u0C83\u0C85-\u0C8C\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBC-\u0CC4\u0CC6-\u0CC8\u0CCA-\u0CCD\u0CD5\u0CD6\u0CDD\u0CDE\u0CE0-\u0CE3\u0CE6-\u0CEF\u0CF1-\u0CF3\u0D00-\u0D0C\u0D0E-\u0D10\u0D12-\u0D44\u0D46-\u0D48\u0D4A-\u0D4E\u0D54-\u0D63\u0D66-\u0D78\u0D7A-\u0D7F\u0D81-\u0D83\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6\u0DCA\u0DCF-\u0DD4\u0DD6\u0DD8-\u0DDF\u0DE6-\u0DEF\u0DF2\u0DF3\u0E01-\u0E3A\u0E40-\u0E4E\u0E50-\u0E59\u0E81\u0E82\u0E84\u0E86-\u0E8A\u0E8C-\u0EA3\u0EA5\u0EA7-\u0EBD\u0EC0-\u0EC4\u0EC6\u0EC8-\u0ECE\u0ED0-\u0ED9\u0EDC-\u0EDF\u0F00\u0F18\u0F19\u0F20-\u0F33\u0F35\u0F37\u0F39\u0F3E-\u0F47\u0F49-\u0F6C\u0F71-\u0F84\u0F86-\u0F97\u0F99-\u0FBC\u0FC6\u1000-\u1049\u1050-\u109D\u10A0-\u10C5\u10C7\u10CD\u10D0-\u10FA\u10FC-\u1248\u124A-\u124D\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310\u1312-\u1315\u1318-\u135A\u135D-\u135F\u1369-\u137C\u1380-\u138F\u13A0-\u13F5\u13F8-\u13FD\u1401-\u166C\u166F-\u167F\u1681-\u169A\u16A0-\u16EA\u16EE-\u16F8\u1700-\u1715\u171F-\u1734\u1740-\u1753\u1760-\u176C\u176E-\u1770\u1772\u1773\u1780-\u17D3\u17D7\u17DC\u17DD\u17E0-\u17E9\u17F0-\u17F9\u180B-\u180D\u180F-\u1819\u1820-\u1878\u1880-\u18AA\u18B0-\u18F5\u1900-\u191E\u1920-\u192B\u1930-\u193B\u1946-\u196D\u1970-\u1974\u1980-\u19AB\u19B0-\u19C9\u19D0-\u19DA\u1A00-\u1A1B\u1A20-\u1A5E\u1A60-\u1A7C\u1A7F-\u1A89\u1A90-\u1A99\u1AA7\u1AB0-\u1ACE\u1B00-\u1B4C\u1B50-\u1B59\u1B6B-\u1B73\u1B80-\u1BF3\u1C00-\u1C37\u1C40-\u1C49\u1C4D-\u1C7D\u1C80-\u1C88\u1C90-\u1CBA\u1CBD-\u1CBF\u1CD0-\u1CD2\u1CD4-\u1CFA\u1D00-\u1F15\u1F18-\u1F1D\u1F20-\u1F45\u1F48-\u1F4D\u1F50-\u1F57\u1F59\u1F5B\u1F5D\u1F5F-\u1F7D\u1F80-\u1FB4\u1FB6-\u1FBC\u1FBE\u1FC2-\u1FC4\u1FC6-\u1FCC\u1FD0-\u1FD3\u1FD6-\u1FDB\u1FE0-\u1FEC\u1FF2-\u1FF4\u1FF6-\u1FFC\u2070\u2071\u2074-\u2079\u207F-\u2089\u2090-\u209C\u20D0-\u20F0\u2102\u2107\u210A-\u2113\u2115\u2119-\u211D\u2124\u2126\u2128\u212A-\u212D\u212F-\u2139\u213C-\u213F\u2145-\u2149\u214E\u2150-\u2189\u2460-\u249B\u24EA-\u24FF\u2776-\u2793\u2C00-\u2CE4\u2CEB-\u2CF3\u2CFD\u2D00-\u2D25\u2D27\u2D2D\u2D30-\u2D67\u2D6F\u2D7F-\u2D96\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\u2DE0-\u2DFF\u2E2F\u3005-\u3007\u3021-\u302F\u3031-\u3035\u3038-\u303C\u3041-\u3096\u3099\u309A\u309D-\u309F\u30A1-\u30FA\u30FC-\u30FF\u3105-\u312F\u3131-\u318E\u3192-\u3195\u31A0-\u31BF\u31F0-\u31FF\u3220-\u3229\u3248-\u324F\u3251-\u325F\u3280-\u3289\u32B1-\u32BF\u3400-\u4DBF\u4E00-\uA48C\uA4D0-\uA4FD\uA500-\uA60C\uA610-\uA62B\uA640-\uA672\uA674-\uA67D\uA67F-\uA6F1\uA717-\uA71F\uA722-\uA788\uA78B-\uA7CA\uA7D0\uA7D1\uA7D3\uA7D5-\uA7D9\uA7F2-\uA827\uA82C\uA830-\uA835\uA840-\uA873\uA880-\uA8C5\uA8D0-\uA8D9\uA8E0-\uA8F7\uA8FB\uA8FD-\uA92D\uA930-\uA953\uA960-\uA97C\uA980-\uA9C0\uA9CF-\uA9D9\uA9E0-\uA9FE\uAA00-\uAA36\uAA40-\uAA4D\uAA50-\uAA59\uAA60-\uAA76\uAA7A-\uAAC2\uAADB-\uAADD\uAAE0-\uAAEF\uAAF2-\uAAF6\uAB01-\uAB06\uAB09-\uAB0E\uAB11-\uAB16\uAB20-\uAB26\uAB28-\uAB2E\uAB30-\uAB5A\uAB5C-\uAB69\uAB70-\uABEA\uABEC\uABED\uABF0-\uABF9\uAC00-\uD7A3\uD7B0-\uD7C6\uD7CB-\uD7FB\uF900-\uFA6D\uFA70-\uFAD9\uFB00-\uFB06\uFB13-\uFB17\uFB1D-\uFB28\uFB2A-\uFB36\uFB38-\uFB3C\uFB3E\uFB40\uFB41\uFB43\uFB44\uFB46-\uFBB1\uFBD3-\uFD3D\uFD50-\uFLine truncated
|
||||
export const whiteSpace =
|
||||
/[\t-\r \u0085\u00A0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000]/
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
/**
|
||||
* Create a new parser.
|
||||
*/
|
||||
export class ParseLatin {
|
||||
/**
|
||||
* Create a new parser.
|
||||
*
|
||||
* This additionally supports `retext`-like call: where an instance is
|
||||
* created for each file, and the file is given on construction.
|
||||
*
|
||||
* @param {string | null | undefined} [doc]
|
||||
* Value to parse (optional).
|
||||
* @param {VFile | null | undefined} [file]
|
||||
* Corresponding file (optional).
|
||||
*/
|
||||
constructor(doc?: string | null | undefined, file?: VFile | null | undefined);
|
||||
/** @type {string | undefined} */
|
||||
doc: string | undefined;
|
||||
/** @type {Array<Plugin<Root>>} */
|
||||
tokenizeRootPlugins: Array<Plugin<Root>>;
|
||||
/** @type {Array<Plugin<Paragraph>>} */
|
||||
tokenizeParagraphPlugins: Array<Plugin<Paragraph>>;
|
||||
/** @type {Array<Plugin<Sentence>>} */
|
||||
tokenizeSentencePlugins: Array<Plugin<Sentence>>;
|
||||
/**
|
||||
* Turn natural language into a syntax tree.
|
||||
*
|
||||
* @param {string | null | undefined} [value]
|
||||
* Value to parse (optional).
|
||||
* @returns {Root}
|
||||
* Tree.
|
||||
*/
|
||||
parse(value?: string | null | undefined): Root;
|
||||
/**
|
||||
* Parse as a root.
|
||||
*
|
||||
* @param {string | null | undefined} [value]
|
||||
* Value to parse (optional).
|
||||
* @returns {Root}
|
||||
* Built tree.
|
||||
*/
|
||||
tokenizeRoot(value?: string | null | undefined): Root;
|
||||
/**
|
||||
* Parse as a paragraph.
|
||||
*
|
||||
* @param {string | null | undefined} [value]
|
||||
* Value to parse (optional).
|
||||
* @returns {Paragraph}
|
||||
* Built tree.
|
||||
*/
|
||||
tokenizeParagraph(value?: string | null | undefined): Paragraph;
|
||||
/**
|
||||
* Parse as a sentence.
|
||||
*
|
||||
* @param {string | null | undefined} [value]
|
||||
* Value to parse (optional).
|
||||
* @returns {Sentence}
|
||||
* Built tree.
|
||||
*/
|
||||
tokenizeSentence(value?: string | null | undefined): Sentence;
|
||||
/**
|
||||
* Transform a `value` into a list of nlcsts.
|
||||
*
|
||||
* @param {string | null | undefined} [value]
|
||||
* Value to parse (optional).
|
||||
* @returns {Array<SentenceContent>}
|
||||
* Built sentence content.
|
||||
*/
|
||||
tokenize(value?: string | null | undefined): Array<SentenceContent>;
|
||||
}
|
||||
export type Nodes = import('nlcst').Nodes;
|
||||
export type Parents = import('nlcst').Parents;
|
||||
export type Paragraph = import('nlcst').Paragraph;
|
||||
export type Root = import('nlcst').Root;
|
||||
export type RootContent = import('nlcst').RootContent;
|
||||
export type Sentence = import('nlcst').Sentence;
|
||||
export type SentenceContent = import('nlcst').SentenceContent;
|
||||
export type VFile = import('vfile').VFile;
|
||||
/**
|
||||
* Transform a node.
|
||||
*/
|
||||
export type Plugin<Node extends import("nlcst").Nodes> = (node: Node) => undefined | void;
|
||||
+352
@@ -0,0 +1,352 @@
|
||||
/**
|
||||
* @typedef {import('nlcst').Nodes} Nodes
|
||||
* @typedef {import('nlcst').Parents} Parents
|
||||
* @typedef {import('nlcst').Paragraph} Paragraph
|
||||
* @typedef {import('nlcst').Root} Root
|
||||
* @typedef {import('nlcst').RootContent} RootContent
|
||||
* @typedef {import('nlcst').Sentence} Sentence
|
||||
* @typedef {import('nlcst').SentenceContent} SentenceContent
|
||||
* @typedef {import('vfile').VFile} VFile
|
||||
*/
|
||||
|
||||
/**
|
||||
* @template {Nodes} Node
|
||||
* Node type.
|
||||
* @callback Plugin
|
||||
* Transform a node.
|
||||
* @param {Node} node
|
||||
* The node.
|
||||
* @returns {undefined | void}
|
||||
* Nothing.
|
||||
*/
|
||||
|
||||
import {toString} from 'nlcst-to-string'
|
||||
import {mergeAffixExceptions} from './plugin/merge-affix-exceptions.js'
|
||||
import {mergeAffixSymbol} from './plugin/merge-affix-symbol.js'
|
||||
import {breakImplicitSentences} from './plugin/break-implicit-sentences.js'
|
||||
import {makeFinalWhiteSpaceSiblings} from './plugin/make-final-white-space-siblings.js'
|
||||
import {makeInitialWhiteSpaceSiblings} from './plugin/make-initial-white-space-siblings.js'
|
||||
import {mergeFinalWordSymbol} from './plugin/merge-final-word-symbol.js'
|
||||
import {mergeInitialDigitSentences} from './plugin/merge-initial-digit-sentences.js'
|
||||
import {mergeInitialLowerCaseLetterSentences} from './plugin/merge-initial-lower-case-letter-sentences.js'
|
||||
import {mergeInitialWordSymbol} from './plugin/merge-initial-word-symbol.js'
|
||||
import {mergeInitialisms} from './plugin/merge-initialisms.js'
|
||||
import {mergeInnerWordSymbol} from './plugin/merge-inner-word-symbol.js'
|
||||
import {mergeInnerWordSlash} from './plugin/merge-inner-word-slash.js'
|
||||
import {mergeNonWordSentences} from './plugin/merge-non-word-sentences.js'
|
||||
import {mergePrefixExceptions} from './plugin/merge-prefix-exceptions.js'
|
||||
import {mergeRemainingFullStops} from './plugin/merge-remaining-full-stops.js'
|
||||
import {removeEmptyNodes} from './plugin/remove-empty-nodes.js'
|
||||
import {patchPosition} from './plugin/patch-position.js'
|
||||
import {
|
||||
newLine,
|
||||
punctuation,
|
||||
surrogates,
|
||||
terminalMarker,
|
||||
whiteSpace,
|
||||
word
|
||||
} from './expressions.js'
|
||||
|
||||
// PARSE LATIN
|
||||
|
||||
/**
|
||||
* Create a new parser.
|
||||
*/
|
||||
export class ParseLatin {
|
||||
/**
|
||||
* Create a new parser.
|
||||
*
|
||||
* This additionally supports `retext`-like call: where an instance is
|
||||
* created for each file, and the file is given on construction.
|
||||
*
|
||||
* @param {string | null | undefined} [doc]
|
||||
* Value to parse (optional).
|
||||
* @param {VFile | null | undefined} [file]
|
||||
* Corresponding file (optional).
|
||||
*/
|
||||
constructor(doc, file) {
|
||||
const value = file || doc
|
||||
|
||||
/** @type {string | undefined} */
|
||||
this.doc = value ? String(value) : undefined
|
||||
|
||||
/** @type {Array<Plugin<Root>>} */
|
||||
this.tokenizeRootPlugins = [...this.tokenizeRootPlugins]
|
||||
/** @type {Array<Plugin<Paragraph>>} */
|
||||
this.tokenizeParagraphPlugins = [...this.tokenizeParagraphPlugins]
|
||||
/** @type {Array<Plugin<Sentence>>} */
|
||||
this.tokenizeSentencePlugins = [...this.tokenizeSentencePlugins]
|
||||
}
|
||||
|
||||
/**
|
||||
* Turn natural language into a syntax tree.
|
||||
*
|
||||
* @param {string | null | undefined} [value]
|
||||
* Value to parse (optional).
|
||||
* @returns {Root}
|
||||
* Tree.
|
||||
*/
|
||||
parse(value) {
|
||||
return this.tokenizeRoot(value || this.doc)
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse as a root.
|
||||
*
|
||||
* @param {string | null | undefined} [value]
|
||||
* Value to parse (optional).
|
||||
* @returns {Root}
|
||||
* Built tree.
|
||||
*/
|
||||
tokenizeRoot(value) {
|
||||
const paragraph = this.tokenizeParagraph(value)
|
||||
/** @type {Root} */
|
||||
const result = {
|
||||
type: 'RootNode',
|
||||
children: splitNode(paragraph, 'WhiteSpaceNode', newLine)
|
||||
}
|
||||
|
||||
let index = -1
|
||||
while (this.tokenizeRootPlugins[++index]) {
|
||||
this.tokenizeRootPlugins[index](result)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse as a paragraph.
|
||||
*
|
||||
* @param {string | null | undefined} [value]
|
||||
* Value to parse (optional).
|
||||
* @returns {Paragraph}
|
||||
* Built tree.
|
||||
*/
|
||||
tokenizeParagraph(value) {
|
||||
const sentence = this.tokenizeSentence(value)
|
||||
/** @type {Paragraph} */
|
||||
const result = {
|
||||
type: 'ParagraphNode',
|
||||
children: splitNode(sentence, 'PunctuationNode', terminalMarker)
|
||||
}
|
||||
|
||||
let index = -1
|
||||
while (this.tokenizeParagraphPlugins[++index]) {
|
||||
this.tokenizeParagraphPlugins[index](result)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse as a sentence.
|
||||
*
|
||||
* @param {string | null | undefined} [value]
|
||||
* Value to parse (optional).
|
||||
* @returns {Sentence}
|
||||
* Built tree.
|
||||
*/
|
||||
tokenizeSentence(value) {
|
||||
const children = this.tokenize(value)
|
||||
/** @type {Sentence} */
|
||||
const result = {type: 'SentenceNode', children}
|
||||
|
||||
let index = -1
|
||||
while (this.tokenizeSentencePlugins[++index]) {
|
||||
this.tokenizeSentencePlugins[index](result)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Transform a `value` into a list of nlcsts.
|
||||
*
|
||||
* @param {string | null | undefined} [value]
|
||||
* Value to parse (optional).
|
||||
* @returns {Array<SentenceContent>}
|
||||
* Built sentence content.
|
||||
*/
|
||||
tokenize(value) {
|
||||
/** @type {Array<SentenceContent>} */
|
||||
const children = []
|
||||
|
||||
if (!value) {
|
||||
return children
|
||||
}
|
||||
|
||||
const currentPoint = {line: 1, column: 1, offset: 0}
|
||||
let from = 0
|
||||
let index = 0
|
||||
let start = {...currentPoint}
|
||||
/** @type {SentenceContent['type'] | undefined} */
|
||||
let previousType
|
||||
/** @type {string | undefined} */
|
||||
let previous
|
||||
|
||||
while (index < value.length) {
|
||||
const current = value.charAt(index)
|
||||
const currentType = whiteSpace.test(current)
|
||||
? 'WhiteSpaceNode'
|
||||
: punctuation.test(current)
|
||||
? 'PunctuationNode'
|
||||
: word.test(current)
|
||||
? 'WordNode'
|
||||
: 'SymbolNode'
|
||||
|
||||
if (
|
||||
from < index &&
|
||||
previousType &&
|
||||
currentType &&
|
||||
!(
|
||||
previousType === currentType &&
|
||||
// Words or white space continue.
|
||||
(previousType === 'WordNode' ||
|
||||
previousType === 'WhiteSpaceNode' ||
|
||||
// Same character of punctuation or symbol also continues.
|
||||
current === previous ||
|
||||
// Surrogates of punctuation or symbol also continue.
|
||||
surrogates.test(current))
|
||||
)
|
||||
) {
|
||||
// Flush the previous queue.
|
||||
children.push(createNode(previousType, value.slice(from, index)))
|
||||
from = index
|
||||
start = {...currentPoint}
|
||||
}
|
||||
|
||||
if (current === '\r' || (current === '\n' && previous !== '\r')) {
|
||||
currentPoint.line++
|
||||
currentPoint.column = 1
|
||||
} else if (current !== '\n') {
|
||||
currentPoint.column++
|
||||
}
|
||||
|
||||
currentPoint.offset++
|
||||
previousType = currentType
|
||||
previous = current
|
||||
index++
|
||||
}
|
||||
|
||||
if (previousType && from < index) {
|
||||
children.push(createNode(previousType, value.slice(from, index)))
|
||||
}
|
||||
|
||||
return children
|
||||
|
||||
/**
|
||||
* @param {SentenceContent['type']} type
|
||||
* Node type to build.
|
||||
* @param {string} value
|
||||
* Value.
|
||||
* @returns {SentenceContent}
|
||||
* Node.
|
||||
*/
|
||||
function createNode(type, value) {
|
||||
return type === 'WordNode'
|
||||
? {
|
||||
type: 'WordNode',
|
||||
children: [
|
||||
{
|
||||
type: 'TextNode',
|
||||
value,
|
||||
position: {start, end: {...currentPoint}}
|
||||
}
|
||||
],
|
||||
position: {start, end: {...currentPoint}}
|
||||
}
|
||||
: {type, value, position: {start, end: {...currentPoint}}}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List of transforms handling a sentence.
|
||||
*/
|
||||
ParseLatin.prototype.tokenizeSentencePlugins = [
|
||||
mergeInitialWordSymbol,
|
||||
mergeFinalWordSymbol,
|
||||
mergeInnerWordSymbol,
|
||||
mergeInnerWordSlash,
|
||||
mergeInitialisms,
|
||||
patchPosition
|
||||
]
|
||||
|
||||
/**
|
||||
* List of transforms handling a paragraph.
|
||||
*/
|
||||
ParseLatin.prototype.tokenizeParagraphPlugins = [
|
||||
mergeNonWordSentences,
|
||||
mergeAffixSymbol,
|
||||
mergeInitialLowerCaseLetterSentences,
|
||||
mergeInitialDigitSentences,
|
||||
mergePrefixExceptions,
|
||||
mergeAffixExceptions,
|
||||
mergeRemainingFullStops,
|
||||
makeInitialWhiteSpaceSiblings,
|
||||
makeFinalWhiteSpaceSiblings,
|
||||
breakImplicitSentences,
|
||||
removeEmptyNodes,
|
||||
patchPosition
|
||||
]
|
||||
|
||||
/**
|
||||
* List of transforms handling a root.
|
||||
*/
|
||||
ParseLatin.prototype.tokenizeRootPlugins = [
|
||||
makeInitialWhiteSpaceSiblings,
|
||||
makeFinalWhiteSpaceSiblings,
|
||||
removeEmptyNodes,
|
||||
patchPosition
|
||||
]
|
||||
|
||||
/**
|
||||
* A function that splits one node into several nodes.
|
||||
*
|
||||
* @template {Parents} Node
|
||||
* Node type.
|
||||
* @param {Node} node
|
||||
* Node to split.
|
||||
* @param {RegExp} expression
|
||||
* Split on this regex.
|
||||
* @param {Node['children'][number]['type']} childType
|
||||
* Split this node type.
|
||||
* @returns {Array<Node>}
|
||||
* The given node, split into several nodes.
|
||||
*/
|
||||
function splitNode(node, childType, expression) {
|
||||
/** @type {Array<Node>} */
|
||||
const result = []
|
||||
let index = -1
|
||||
let start = 0
|
||||
|
||||
while (++index < node.children.length) {
|
||||
const token = node.children[index]
|
||||
|
||||
if (
|
||||
index === node.children.length - 1 ||
|
||||
(token.type === childType && expression.test(toString(token)))
|
||||
) {
|
||||
/** @type {Node} */
|
||||
// @ts-expect-error: fine
|
||||
const parent = {
|
||||
type: node.type,
|
||||
children: node.children.slice(start, index + 1)
|
||||
}
|
||||
|
||||
const first = node.children[start]
|
||||
const last = token
|
||||
if (first.position && last.position) {
|
||||
parent.position = {
|
||||
start: first.position.start,
|
||||
end: last.position.end
|
||||
}
|
||||
}
|
||||
|
||||
result.push(parent)
|
||||
start = index + 1
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
export const breakImplicitSentences: import("../../node_modules/unist-util-modify-children/lib/index.js").Modify<import("nlcst").Paragraph>;
|
||||
export type Paragraph = import('nlcst').Paragraph;
|
||||
export type Sentence = import('nlcst').Sentence;
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
/**
|
||||
* @typedef {import('nlcst').Paragraph} Paragraph
|
||||
* @typedef {import('nlcst').Sentence} Sentence
|
||||
*/
|
||||
|
||||
import {toString} from 'nlcst-to-string'
|
||||
import {modifyChildren} from 'unist-util-modify-children'
|
||||
|
||||
// Break a sentence if a white space with more than one new-line is found.
|
||||
export const breakImplicitSentences = modifyChildren(
|
||||
/**
|
||||
* @type {import('unist-util-modify-children').Modifier<Paragraph>}
|
||||
*/
|
||||
|
||||
function (child, index, parent) {
|
||||
if (child.type !== 'SentenceNode') {
|
||||
return
|
||||
}
|
||||
|
||||
const children = child.children
|
||||
|
||||
// Ignore first and last child.
|
||||
let position = 0
|
||||
|
||||
while (++position < children.length - 1) {
|
||||
const node = children[position]
|
||||
|
||||
if (
|
||||
node.type !== 'WhiteSpaceNode' ||
|
||||
toString(node).split(/\r\n|\r|\n/).length < 3
|
||||
) {
|
||||
continue
|
||||
}
|
||||
|
||||
child.children = children.slice(0, position)
|
||||
|
||||
/** @type {Sentence} */
|
||||
const insertion = {
|
||||
type: 'SentenceNode',
|
||||
children: children.slice(position + 1)
|
||||
}
|
||||
|
||||
const tail = children[position - 1]
|
||||
const head = children[position + 1]
|
||||
|
||||
parent.children.splice(index + 1, 0, node, insertion)
|
||||
|
||||
if (child.position && tail.position && head.position) {
|
||||
const end = child.position.end
|
||||
|
||||
child.position.end = tail.position.end
|
||||
|
||||
insertion.position = {start: head.position.start, end}
|
||||
}
|
||||
|
||||
return index + 1
|
||||
}
|
||||
}
|
||||
)
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
export const makeFinalWhiteSpaceSiblings: import("../../node_modules/unist-util-modify-children/lib/index.js").Modify<import("nlcst").Paragraph | import("nlcst").Root>;
|
||||
export type Paragraph = import('nlcst').Paragraph;
|
||||
export type Root = import('nlcst').Root;
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
/**
|
||||
* @typedef {import('nlcst').Paragraph} Paragraph
|
||||
* @typedef {import('nlcst').Root} Root
|
||||
*/
|
||||
|
||||
import {modifyChildren} from 'unist-util-modify-children'
|
||||
|
||||
// Move white space ending a paragraph up, so they are the siblings of
|
||||
// paragraphs.
|
||||
export const makeFinalWhiteSpaceSiblings = modifyChildren(
|
||||
/**
|
||||
* @type {import('unist-util-modify-children').Modifier<Paragraph | Root>}
|
||||
*/
|
||||
|
||||
function (child, index, parent) {
|
||||
if ('children' in child) {
|
||||
const tail = child.children[child.children.length - 1]
|
||||
|
||||
if (tail && tail.type === 'WhiteSpaceNode') {
|
||||
child.children.pop() // Remove `tail`.
|
||||
parent.children.splice(index + 1, 0, tail)
|
||||
const previous = child.children[child.children.length - 1]
|
||||
|
||||
if (previous && previous.position && child.position) {
|
||||
child.position.end = previous.position.end
|
||||
}
|
||||
|
||||
// Next, iterate over the current node again.
|
||||
return index
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
export const makeInitialWhiteSpaceSiblings: import("../../node_modules/unist-util-visit-children/lib/index.js").Visit<import("nlcst").Paragraph | import("nlcst").Root>;
|
||||
export type Paragraph = import('nlcst').Paragraph;
|
||||
export type Root = import('nlcst').Root;
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
/**
|
||||
* @typedef {import('nlcst').Paragraph} Paragraph
|
||||
* @typedef {import('nlcst').Root} Root
|
||||
*/
|
||||
|
||||
import {visitChildren} from 'unist-util-visit-children'
|
||||
|
||||
// Move white space starting a sentence up, so they are the siblings of
|
||||
// sentences.
|
||||
export const makeInitialWhiteSpaceSiblings = visitChildren(
|
||||
/**
|
||||
* @type {import('unist-util-visit-children').Visitor<Paragraph | Root>}
|
||||
*/
|
||||
function (child, index, parent) {
|
||||
if ('children' in child && child.children) {
|
||||
const head = child.children[0]
|
||||
if (head && head.type === 'WhiteSpaceNode') {
|
||||
child.children.shift()
|
||||
parent.children.splice(index, 0, head)
|
||||
const next = child.children[0]
|
||||
|
||||
if (next && next.position && child.position) {
|
||||
child.position.start = next.position.start
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
export const mergeAffixExceptions: import("../../node_modules/unist-util-modify-children/lib/index.js").Modify<import("nlcst").Paragraph>;
|
||||
export type Paragraph = import('nlcst').Paragraph;
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
/**
|
||||
* @typedef {import('nlcst').Paragraph} Paragraph
|
||||
*/
|
||||
|
||||
import {toString} from 'nlcst-to-string'
|
||||
import {modifyChildren} from 'unist-util-modify-children'
|
||||
|
||||
// Merge a sentence into its previous sentence, when the sentence starts with a
|
||||
// comma.
|
||||
export const mergeAffixExceptions = modifyChildren(
|
||||
/**
|
||||
* @type {import('unist-util-modify-children').Modifier<Paragraph>}
|
||||
*/
|
||||
function (child, index, parent) {
|
||||
const previous = parent.children[index - 1]
|
||||
|
||||
if (
|
||||
previous &&
|
||||
'children' in previous &&
|
||||
'children' in child &&
|
||||
child.children.length > 0
|
||||
) {
|
||||
let position = -1
|
||||
|
||||
while (child.children[++position]) {
|
||||
const node = child.children[position]
|
||||
|
||||
if (node.type === 'WordNode') {
|
||||
return
|
||||
}
|
||||
|
||||
if (node.type === 'SymbolNode' || node.type === 'PunctuationNode') {
|
||||
const value = toString(node)
|
||||
|
||||
if (value !== ',' && value !== ';') {
|
||||
return
|
||||
}
|
||||
|
||||
previous.children.push(...child.children)
|
||||
|
||||
// Update position.
|
||||
if (previous.position && child.position) {
|
||||
previous.position.end = child.position.end
|
||||
}
|
||||
|
||||
parent.children.splice(index, 1)
|
||||
|
||||
// Next, iterate over the node *now* at the current position.
|
||||
return index
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
export const mergeAffixSymbol: import("../../node_modules/unist-util-modify-children/lib/index.js").Modify<import("nlcst").Paragraph>;
|
||||
export type Paragraph = import('nlcst').Paragraph;
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
/**
|
||||
* @typedef {import('nlcst').Paragraph} Paragraph
|
||||
*/
|
||||
|
||||
import {toString} from 'nlcst-to-string'
|
||||
import {modifyChildren} from 'unist-util-modify-children'
|
||||
// Closing or final punctuation, or terminal markers that should still be
|
||||
// included in the previous sentence, even though they follow the sentence’s
|
||||
// terminal marker.
|
||||
import {affixSymbol} from '../expressions.js'
|
||||
|
||||
// Move certain punctuation following a terminal marker (thus in the next
|
||||
// sentence) to the previous sentence.
|
||||
export const mergeAffixSymbol = modifyChildren(
|
||||
/**
|
||||
* @type {import('unist-util-modify-children').Modifier<Paragraph>}
|
||||
*/
|
||||
function (child, index, parent) {
|
||||
if ('children' in child && child.children.length > 0 && index > 0) {
|
||||
const previous = parent.children[index - 1]
|
||||
const first = child.children[0]
|
||||
const second = child.children[1]
|
||||
|
||||
if (
|
||||
previous &&
|
||||
previous.type === 'SentenceNode' &&
|
||||
(first.type === 'SymbolNode' || first.type === 'PunctuationNode') &&
|
||||
affixSymbol.test(toString(first))
|
||||
) {
|
||||
child.children.shift() // Remove `first`.
|
||||
previous.children.push(first)
|
||||
|
||||
// Update position.
|
||||
if (first.position && previous.position) {
|
||||
previous.position.end = first.position.end
|
||||
}
|
||||
|
||||
if (second && second.position && child.position) {
|
||||
child.position.start = second.position.start
|
||||
}
|
||||
|
||||
// Next, iterate over the previous node again.
|
||||
return index - 1
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
export const mergeFinalWordSymbol: import("../../node_modules/unist-util-modify-children/lib/index.js").Modify<import("nlcst").Sentence>;
|
||||
export type Sentence = import('nlcst').Sentence;
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
/**
|
||||
* @typedef {import('nlcst').Sentence} Sentence
|
||||
*/
|
||||
|
||||
import {toString} from 'nlcst-to-string'
|
||||
import {modifyChildren} from 'unist-util-modify-children'
|
||||
|
||||
// Merge certain punctuation marks into their preceding words.
|
||||
export const mergeFinalWordSymbol = modifyChildren(
|
||||
/**
|
||||
* @type {import('unist-util-modify-children').Modifier<Sentence>}
|
||||
*/
|
||||
function (child, index, parent) {
|
||||
if (
|
||||
index > 0 &&
|
||||
(child.type === 'SymbolNode' || child.type === 'PunctuationNode') &&
|
||||
toString(child) === '-'
|
||||
) {
|
||||
const children = parent.children
|
||||
const previous = children[index - 1]
|
||||
const next = children[index + 1]
|
||||
|
||||
if (
|
||||
(!next || next.type !== 'WordNode') &&
|
||||
previous &&
|
||||
previous.type === 'WordNode'
|
||||
) {
|
||||
// Remove `child` from parent.
|
||||
children.splice(index, 1)
|
||||
|
||||
// Add the punctuation mark at the end of the previous node.
|
||||
previous.children.push(child)
|
||||
|
||||
// Update position.
|
||||
if (previous.position && child.position) {
|
||||
previous.position.end = child.position.end
|
||||
}
|
||||
|
||||
// Next, iterate over the node *now* at the current position (which was
|
||||
// the next node).
|
||||
return index
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
export const mergeInitialDigitSentences: import("../../node_modules/unist-util-modify-children/lib/index.js").Modify<import("nlcst").Paragraph>;
|
||||
export type Paragraph = import('nlcst').Paragraph;
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
/**
|
||||
* @typedef {import('nlcst').Paragraph} Paragraph
|
||||
*/
|
||||
|
||||
import {toString} from 'nlcst-to-string'
|
||||
import {modifyChildren} from 'unist-util-modify-children'
|
||||
import {digitStart} from '../expressions.js'
|
||||
|
||||
// Merge a sentence into its previous sentence, when the sentence starts with a
|
||||
// lower case letter.
|
||||
export const mergeInitialDigitSentences = modifyChildren(
|
||||
/**
|
||||
* @type {import('unist-util-modify-children').Modifier<Paragraph>}
|
||||
*/
|
||||
function (child, index, parent) {
|
||||
const previous = parent.children[index - 1]
|
||||
|
||||
if (
|
||||
previous &&
|
||||
previous.type === 'SentenceNode' &&
|
||||
child.type === 'SentenceNode'
|
||||
) {
|
||||
const head = child.children[0]
|
||||
|
||||
if (head && head.type === 'WordNode' && digitStart.test(toString(head))) {
|
||||
previous.children.push(...child.children)
|
||||
parent.children.splice(index, 1)
|
||||
|
||||
// Update position.
|
||||
if (previous.position && child.position) {
|
||||
previous.position.end = child.position.end
|
||||
}
|
||||
|
||||
// Next, iterate over the node *now* at the current position.
|
||||
return index
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
Generated
Vendored
+2
@@ -0,0 +1,2 @@
|
||||
export const mergeInitialLowerCaseLetterSentences: import("../../node_modules/unist-util-modify-children/lib/index.js").Modify<import("nlcst").Paragraph>;
|
||||
export type Paragraph = import('nlcst').Paragraph;
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
/**
|
||||
* @typedef {import('nlcst').Paragraph} Paragraph
|
||||
*/
|
||||
|
||||
import {toString} from 'nlcst-to-string'
|
||||
import {modifyChildren} from 'unist-util-modify-children'
|
||||
// Initial lowercase letter.
|
||||
import {lowerInitial} from '../expressions.js'
|
||||
|
||||
// Merge a sentence into its previous sentence, when the sentence starts with a
|
||||
// lower case letter.
|
||||
export const mergeInitialLowerCaseLetterSentences = modifyChildren(
|
||||
/**
|
||||
* @type {import('unist-util-modify-children').Modifier<Paragraph>}
|
||||
*/
|
||||
function (child, index, parent) {
|
||||
if (child.type === 'SentenceNode' && index > 0) {
|
||||
const previous = parent.children[index - 1]
|
||||
const children = child.children
|
||||
|
||||
if (children.length > 0 && previous.type === 'SentenceNode') {
|
||||
let position = -1
|
||||
|
||||
while (children[++position]) {
|
||||
const node = children[position]
|
||||
|
||||
if (node.type === 'WordNode') {
|
||||
if (!lowerInitial.test(toString(node))) {
|
||||
return
|
||||
}
|
||||
|
||||
previous.children.push(...children)
|
||||
|
||||
parent.children.splice(index, 1)
|
||||
|
||||
// Update position.
|
||||
if (previous.position && child.position) {
|
||||
previous.position.end = child.position.end
|
||||
}
|
||||
|
||||
// Next, iterate over the node *now* at the current position.
|
||||
return index
|
||||
}
|
||||
|
||||
if (node.type === 'SymbolNode' || node.type === 'PunctuationNode') {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
export const mergeInitialWordSymbol: import("../../node_modules/unist-util-modify-children/lib/index.js").Modify<import("nlcst").Sentence>;
|
||||
export type Sentence = import('nlcst').Sentence;
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
/**
|
||||
* @typedef {import('nlcst').Sentence} Sentence
|
||||
*/
|
||||
|
||||
import {toString} from 'nlcst-to-string'
|
||||
import {modifyChildren} from 'unist-util-modify-children'
|
||||
|
||||
// Merge certain punctuation marks into their following words.
|
||||
export const mergeInitialWordSymbol = modifyChildren(
|
||||
/**
|
||||
* @type {import('unist-util-modify-children').Modifier<Sentence>}
|
||||
*/
|
||||
function (child, index, parent) {
|
||||
if (
|
||||
(child.type !== 'SymbolNode' && child.type !== 'PunctuationNode') ||
|
||||
toString(child) !== '&'
|
||||
) {
|
||||
return
|
||||
}
|
||||
|
||||
const children = parent.children
|
||||
const next = children[index + 1]
|
||||
|
||||
// If either a previous word, or no following word, exists, exit early.
|
||||
if (
|
||||
(index > 0 && children[index - 1].type === 'WordNode') ||
|
||||
!(next && next.type === 'WordNode')
|
||||
) {
|
||||
return
|
||||
}
|
||||
|
||||
// Remove `child` from parent.
|
||||
children.splice(index, 1)
|
||||
|
||||
// Add the punctuation mark at the start of the next node.
|
||||
next.children.unshift(child)
|
||||
|
||||
// Update position.
|
||||
if (next.position && child.position) {
|
||||
next.position.start = child.position.start
|
||||
}
|
||||
|
||||
// Next, iterate over the node at the previous position, as it's now adjacent
|
||||
// to a following word.
|
||||
return index - 1
|
||||
}
|
||||
)
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
export const mergeInitialisms: import("../../node_modules/unist-util-modify-children/lib/index.js").Modify<import("nlcst").Sentence>;
|
||||
export type Sentence = import('nlcst').Sentence;
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
/**
|
||||
* @typedef {import('nlcst').Sentence} Sentence
|
||||
*/
|
||||
|
||||
import {toString} from 'nlcst-to-string'
|
||||
import {modifyChildren} from 'unist-util-modify-children'
|
||||
import {numerical} from '../expressions.js'
|
||||
|
||||
// Merge initialisms.
|
||||
export const mergeInitialisms = modifyChildren(
|
||||
/**
|
||||
* @type {import('unist-util-modify-children').Modifier<Sentence>}
|
||||
*/
|
||||
function (child, index, parent) {
|
||||
if (
|
||||
index > 0 &&
|
||||
child.type === 'PunctuationNode' &&
|
||||
toString(child) === '.'
|
||||
) {
|
||||
const previous = parent.children[index - 1]
|
||||
|
||||
if (
|
||||
previous.type === 'WordNode' &&
|
||||
previous.children &&
|
||||
previous.children.length !== 1 &&
|
||||
previous.children.length % 2 !== 0
|
||||
) {
|
||||
let position = previous.children.length
|
||||
let isAllDigits = true
|
||||
|
||||
while (previous.children[--position]) {
|
||||
const otherChild = previous.children[position]
|
||||
|
||||
const value = toString(otherChild)
|
||||
|
||||
if (position % 2 === 0) {
|
||||
// Initialisms consist of one character values.
|
||||
if (value.length > 1) {
|
||||
return
|
||||
}
|
||||
|
||||
if (!numerical.test(value)) {
|
||||
isAllDigits = false
|
||||
}
|
||||
} else if (value !== '.') {
|
||||
if (position < previous.children.length - 2) {
|
||||
break
|
||||
} else {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!isAllDigits) {
|
||||
// Remove `child` from parent.
|
||||
parent.children.splice(index, 1)
|
||||
|
||||
// Add child to the previous children.
|
||||
previous.children.push(child)
|
||||
|
||||
// Update position.
|
||||
if (previous.position && child.position) {
|
||||
previous.position.end = child.position.end
|
||||
}
|
||||
|
||||
// Next, iterate over the node *now* at the current position.
|
||||
return index
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
export const mergeInnerWordSlash: import("../../node_modules/unist-util-modify-children/lib/index.js").Modify<import("nlcst").Sentence>;
|
||||
export type Sentence = import('nlcst').Sentence;
|
||||
export type SentenceContent = import('nlcst').SentenceContent;
|
||||
export type WordContent = import('nlcst').WordContent;
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
/**
|
||||
* @typedef {import('nlcst').Sentence} Sentence
|
||||
* @typedef {import('nlcst').SentenceContent} SentenceContent
|
||||
* @typedef {import('nlcst').WordContent} WordContent
|
||||
*/
|
||||
|
||||
import {toString} from 'nlcst-to-string'
|
||||
import {modifyChildren} from 'unist-util-modify-children'
|
||||
|
||||
// Merge words joined by certain punctuation marks.
|
||||
export const mergeInnerWordSlash = modifyChildren(
|
||||
/**
|
||||
* @type {import('unist-util-modify-children').Modifier<Sentence>}
|
||||
*/
|
||||
function (child, index, parent) {
|
||||
const siblings = parent.children
|
||||
const previous = siblings[index - 1]
|
||||
|
||||
if (
|
||||
previous &&
|
||||
previous.type === 'WordNode' &&
|
||||
(child.type === 'SymbolNode' || child.type === 'PunctuationNode') &&
|
||||
toString(child) === '/'
|
||||
) {
|
||||
const previousValue = toString(previous)
|
||||
/** @type {SentenceContent} */
|
||||
let tail = child
|
||||
/** @type {Array<WordContent>} */
|
||||
const queue = [child]
|
||||
let count = 1
|
||||
let nextValue = ''
|
||||
const next = siblings[index + 1]
|
||||
|
||||
if (next && next.type === 'WordNode') {
|
||||
nextValue = toString(next)
|
||||
tail = next
|
||||
queue.push(...next.children)
|
||||
count++
|
||||
}
|
||||
|
||||
if (previousValue.length < 3 && (!nextValue || nextValue.length < 3)) {
|
||||
// Add all found tokens to `prev`s children.
|
||||
previous.children.push(...queue)
|
||||
|
||||
siblings.splice(index, count)
|
||||
|
||||
// Update position.
|
||||
if (previous.position && tail.position) {
|
||||
previous.position.end = tail.position.end
|
||||
}
|
||||
|
||||
// Next, iterate over the node *now* at the current position.
|
||||
return index
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
export const mergeInnerWordSymbol: import("../../node_modules/unist-util-modify-children/lib/index.js").Modify<import("nlcst").Sentence>;
|
||||
export type Sentence = import('nlcst').Sentence;
|
||||
export type WordContent = import('nlcst').WordContent;
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
/**
|
||||
* @typedef {import('nlcst').Sentence} Sentence
|
||||
* @typedef {import('nlcst').WordContent} WordContent
|
||||
*/
|
||||
|
||||
import {toString} from 'nlcst-to-string'
|
||||
import {modifyChildren} from 'unist-util-modify-children'
|
||||
// Symbols part of surrounding words.
|
||||
import {wordSymbolInner} from '../expressions.js'
|
||||
|
||||
// Merge words joined by certain punctuation marks.
|
||||
export const mergeInnerWordSymbol = modifyChildren(
|
||||
/**
|
||||
* @type {import('unist-util-modify-children').Modifier<Sentence>}
|
||||
*/
|
||||
function (child, index, parent) {
|
||||
if (
|
||||
index > 0 &&
|
||||
(child.type === 'SymbolNode' || child.type === 'PunctuationNode')
|
||||
) {
|
||||
const siblings = parent.children
|
||||
const previous = siblings[index - 1]
|
||||
|
||||
if (previous && previous.type === 'WordNode') {
|
||||
let position = index - 1
|
||||
/** @type {Array<WordContent>} */
|
||||
const tokens = []
|
||||
/** @type {Array<WordContent>} */
|
||||
let queue = []
|
||||
|
||||
// - If a token which is neither word nor inner word symbol is found,
|
||||
// the loop is broken
|
||||
// - If an inner word symbol is found, it’s queued
|
||||
// - If a word is found, it’s queued (and the queue stored and emptied)
|
||||
while (siblings[++position]) {
|
||||
const sibling = siblings[position]
|
||||
|
||||
if (sibling.type === 'WordNode') {
|
||||
tokens.push(...queue, ...sibling.children)
|
||||
|
||||
queue = []
|
||||
} else if (
|
||||
(sibling.type === 'SymbolNode' ||
|
||||
sibling.type === 'PunctuationNode') &&
|
||||
wordSymbolInner.test(toString(sibling))
|
||||
) {
|
||||
queue.push(sibling)
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if (tokens.length > 0) {
|
||||
// If there is a queue, remove its length from `position`.
|
||||
if (queue.length > 0) {
|
||||
position -= queue.length
|
||||
}
|
||||
|
||||
// Remove every (one or more) inner-word punctuation marks and children
|
||||
// of words.
|
||||
siblings.splice(index, position - index)
|
||||
|
||||
// Add all found tokens to `prev`s children.
|
||||
previous.children.push(...tokens)
|
||||
|
||||
const last = tokens[tokens.length - 1]
|
||||
|
||||
// Update position.
|
||||
if (previous.position && last.position) {
|
||||
previous.position.end = last.position.end
|
||||
}
|
||||
|
||||
// Next, iterate over the node *now* at the current position.
|
||||
return index
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
export const mergeNonWordSentences: import("../../node_modules/unist-util-modify-children/lib/index.js").Modify<import("nlcst").Paragraph>;
|
||||
export type Paragraph = import('nlcst').Paragraph;
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
/**
|
||||
* @typedef {import('nlcst').Paragraph} Paragraph
|
||||
*/
|
||||
|
||||
import {modifyChildren} from 'unist-util-modify-children'
|
||||
|
||||
// Merge a sentence into the following sentence, when the sentence does not
|
||||
// contain word tokens.
|
||||
export const mergeNonWordSentences = modifyChildren(
|
||||
/**
|
||||
* @type {import('unist-util-modify-children').Modifier<Paragraph>}
|
||||
*/
|
||||
function (child, index, parent) {
|
||||
if ('children' in child) {
|
||||
let position = -1
|
||||
|
||||
while (child.children[++position]) {
|
||||
if (child.children[position].type === 'WordNode') {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
const previous = parent.children[index - 1]
|
||||
|
||||
if (previous && 'children' in previous) {
|
||||
previous.children.push(...child.children)
|
||||
|
||||
// Remove the child.
|
||||
parent.children.splice(index, 1)
|
||||
|
||||
// Patch position.
|
||||
if (previous.position && child.position) {
|
||||
previous.position.end = child.position.end
|
||||
}
|
||||
|
||||
// Next, iterate over the node *now* at the current position (which was the
|
||||
// next node).
|
||||
return index
|
||||
}
|
||||
|
||||
const next = parent.children[index + 1]
|
||||
|
||||
if (next && 'children' in next) {
|
||||
next.children.unshift(...child.children)
|
||||
|
||||
// Patch position.
|
||||
if (next.position && child.position) {
|
||||
next.position.start = child.position.start
|
||||
}
|
||||
|
||||
// Remove the child.
|
||||
parent.children.splice(index, 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
export const mergePrefixExceptions: import("../../node_modules/unist-util-modify-children/lib/index.js").Modify<import("nlcst").Paragraph>;
|
||||
export type Paragraph = import('nlcst').Paragraph;
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
/**
|
||||
* @typedef {import('nlcst').Paragraph} Paragraph
|
||||
*/
|
||||
|
||||
import {toString} from 'nlcst-to-string'
|
||||
import {modifyChildren} from 'unist-util-modify-children'
|
||||
|
||||
// Full stop characters that should not be treated as terminal sentence markers:
|
||||
// A case-insensitive abbreviation.
|
||||
const abbreviationPrefix = new RegExp(
|
||||
'^(' +
|
||||
'[0-9]{1,3}|' +
|
||||
'[a-z]|' +
|
||||
// Common Latin Abbreviations:
|
||||
// Based on: <https://en.wikipedia.org/wiki/List_of_Latin_abbreviations>.
|
||||
// Where only the abbreviations written without joining full stops,
|
||||
// but with a final full stop, were extracted.
|
||||
//
|
||||
// circa, capitulus, confer, compare, centum weight, eadem, (et) alii,
|
||||
// et cetera, floruit, foliis, ibidem, idem, nemine && contradicente,
|
||||
// opere && citato, (per) cent, (per) procurationem, (pro) tempore,
|
||||
// sic erat scriptum, (et) sequentia, statim, videlicet. */
|
||||
'al|ca|cap|cca|cent|cf|cit|con|cp|cwt|ead|etc|ff|' +
|
||||
'fl|ibid|id|nem|op|pro|seq|sic|stat|tem|viz' +
|
||||
')$'
|
||||
)
|
||||
|
||||
// Merge a sentence into its next sentence, when the sentence ends with a
|
||||
// certain word.
|
||||
export const mergePrefixExceptions = modifyChildren(
|
||||
/**
|
||||
* @type {import('unist-util-modify-children').Modifier<Paragraph>}
|
||||
*/
|
||||
function (child, index, parent) {
|
||||
if ('children' in child && child.children.length > 1) {
|
||||
const period = child.children[child.children.length - 1]
|
||||
|
||||
if (
|
||||
period &&
|
||||
(period.type === 'PunctuationNode' || period.type === 'SymbolNode') &&
|
||||
toString(period) === '.'
|
||||
) {
|
||||
const node = child.children[child.children.length - 2]
|
||||
|
||||
if (
|
||||
node &&
|
||||
node.type === 'WordNode' &&
|
||||
abbreviationPrefix.test(toString(node).toLowerCase())
|
||||
) {
|
||||
// Merge period into abbreviation.
|
||||
node.children.push(period)
|
||||
child.children.pop()
|
||||
|
||||
// Update position.
|
||||
if (period.position && node.position) {
|
||||
node.position.end = period.position.end
|
||||
}
|
||||
|
||||
// Merge sentences.
|
||||
const next = parent.children[index + 1]
|
||||
|
||||
if (next && next.type === 'SentenceNode') {
|
||||
child.children.push(...next.children)
|
||||
parent.children.splice(index + 1, 1)
|
||||
|
||||
// Update position.
|
||||
if (next.position && child.position) {
|
||||
child.position.end = next.position.end
|
||||
}
|
||||
|
||||
// Next, iterate over the current node again.
|
||||
return index - 1
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
export const mergeRemainingFullStops: import("../../node_modules/unist-util-visit-children/lib/index.js").Visit<import("nlcst").Paragraph>;
|
||||
export type Paragraph = import('nlcst').Paragraph;
|
||||
+99
@@ -0,0 +1,99 @@
|
||||
/**
|
||||
* @typedef {import('nlcst').Paragraph} Paragraph
|
||||
*/
|
||||
|
||||
import {toString} from 'nlcst-to-string'
|
||||
import {visitChildren} from 'unist-util-visit-children'
|
||||
// Full stop characters that should not be treated as terminal sentence markers:
|
||||
// A case-insensitive abbreviation.
|
||||
import {terminalMarker} from '../expressions.js'
|
||||
|
||||
// Merge non-terminal-marker full stops into the previous word (if available),
|
||||
// or the next word (if available).
|
||||
export const mergeRemainingFullStops = visitChildren(
|
||||
/**
|
||||
* @type {import('unist-util-visit-children').Visitor<Paragraph>}
|
||||
*/
|
||||
// eslint-disable-next-line complexity
|
||||
function (child, _, _parent) {
|
||||
if ('children' in child) {
|
||||
let position = child.children.length
|
||||
let hasFoundDelimiter = false
|
||||
|
||||
while (child.children[--position]) {
|
||||
const grandchild = child.children[position]
|
||||
|
||||
if (
|
||||
grandchild.type !== 'SymbolNode' &&
|
||||
grandchild.type !== 'PunctuationNode'
|
||||
) {
|
||||
// This is a sentence without terminal marker, so we 'fool' the code to
|
||||
// make it think we have found one.
|
||||
if (grandchild.type === 'WordNode') {
|
||||
hasFoundDelimiter = true
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
// Exit when this token is not a terminal marker.
|
||||
if (!terminalMarker.test(toString(grandchild))) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Ignore the first terminal marker found (starting at the end), as it
|
||||
// should not be merged.
|
||||
if (!hasFoundDelimiter) {
|
||||
hasFoundDelimiter = true
|
||||
continue
|
||||
}
|
||||
|
||||
// Only merge a single full stop.
|
||||
if (toString(grandchild) !== '.') {
|
||||
continue
|
||||
}
|
||||
|
||||
const previous = child.children[position - 1]
|
||||
const next = child.children[position + 1]
|
||||
|
||||
if (previous && previous.type === 'WordNode') {
|
||||
const nextNext = child.children[position + 2]
|
||||
|
||||
// Continue when the full stop is followed by a space and another full
|
||||
// stop, such as: `{.} .`
|
||||
if (
|
||||
next &&
|
||||
nextNext &&
|
||||
next.type === 'WhiteSpaceNode' &&
|
||||
toString(nextNext) === '.'
|
||||
) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Remove `child` from parent.
|
||||
child.children.splice(position, 1)
|
||||
|
||||
// Add the punctuation mark at the end of the previous node.
|
||||
previous.children.push(grandchild)
|
||||
|
||||
// Update position.
|
||||
if (grandchild.position && previous.position) {
|
||||
previous.position.end = grandchild.position.end
|
||||
}
|
||||
|
||||
position--
|
||||
} else if (next && next.type === 'WordNode') {
|
||||
// Remove `child` from parent.
|
||||
child.children.splice(position, 1)
|
||||
|
||||
// Add the punctuation mark at the start of the next node.
|
||||
next.children.unshift(grandchild)
|
||||
|
||||
if (grandchild.position && next.position) {
|
||||
next.position.start = grandchild.position.start
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
export const patchPosition: import("../../node_modules/unist-util-visit-children/lib/index.js").Visit<import("nlcst").Sentence | import("nlcst").Paragraph | import("nlcst").Root>;
|
||||
export type Node = import('unist').Node;
|
||||
export type Paragraph = import('nlcst').Paragraph;
|
||||
export type Position = import('unist').Position;
|
||||
export type Root = import('nlcst').Root;
|
||||
export type Sentence = import('nlcst').Sentence;
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
/**
|
||||
* @typedef {import('unist').Node} Node
|
||||
* @typedef {import('nlcst').Paragraph} Paragraph
|
||||
* @typedef {import('unist').Position} Position
|
||||
* @typedef {import('nlcst').Root} Root
|
||||
* @typedef {import('nlcst').Sentence} Sentence
|
||||
*/
|
||||
|
||||
import {visitChildren} from 'unist-util-visit-children'
|
||||
|
||||
// Patch the position on a parent node based on its first and last child.
|
||||
export const patchPosition = visitChildren(
|
||||
/**
|
||||
* @type {import('unist-util-visit-children').Visitor<Paragraph | Root | Sentence>}
|
||||
*/
|
||||
function (child, index, node) {
|
||||
const siblings = node.children
|
||||
|
||||
if (
|
||||
child.position &&
|
||||
index < 1 &&
|
||||
/* c8 ignore next */
|
||||
(!node.position || !node.position.start)
|
||||
) {
|
||||
patch(node)
|
||||
node.position.start = child.position.start
|
||||
}
|
||||
|
||||
if (
|
||||
child.position &&
|
||||
index === siblings.length - 1 &&
|
||||
(!node.position || !node.position.end)
|
||||
) {
|
||||
patch(node)
|
||||
node.position.end = child.position.end
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
/**
|
||||
* @param {Node} node
|
||||
* @returns {asserts node is Node & {position: Position}}
|
||||
*/
|
||||
function patch(node) {
|
||||
if (!node.position) {
|
||||
// @ts-expect-error: fine, we’ll fill it later.
|
||||
node.position = {}
|
||||
}
|
||||
}
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
export const removeEmptyNodes: import("../../node_modules/unist-util-modify-children/lib/index.js").Modify<import("nlcst").Paragraph | import("nlcst").Root>;
|
||||
export type Paragraph = import('nlcst').Paragraph;
|
||||
export type Root = import('nlcst').Root;
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
/**
|
||||
* @typedef {import('nlcst').Paragraph} Paragraph
|
||||
* @typedef {import('nlcst').Root} Root
|
||||
*/
|
||||
|
||||
import {modifyChildren} from 'unist-util-modify-children'
|
||||
|
||||
// Remove empty children.
|
||||
export const removeEmptyNodes = modifyChildren(
|
||||
/**
|
||||
* @type {import('unist-util-modify-children').Modifier<Paragraph | Root>}
|
||||
*/
|
||||
|
||||
function (child, index, parent) {
|
||||
if ('children' in child && child.children.length === 0) {
|
||||
parent.children.splice(index, 1)
|
||||
|
||||
// Next, iterate over the node *now* at the current position (which was the
|
||||
// next node).
|
||||
return index
|
||||
}
|
||||
}
|
||||
)
|
||||
Reference in new issue
Block a user