diff --git a/.gitignore b/.gitignore index c4b000a..ca1d866 100644 --- a/.gitignore +++ b/.gitignore @@ -108,13 +108,15 @@ venv.bak/ # mypy .mypy_cache/ -#notepad++ backups -nppBackup .dmypy.json dmypy.json +#notepad++ backups +nppBackup + # Pyre type checker .pyre/ # HTML files for testing *.html +!**/uBlock/* diff --git a/restscrape/__init__.py b/restscrape/__init__.py index e69de29..78051f3 100644 --- a/restscrape/__init__.py +++ b/restscrape/__init__.py @@ -0,0 +1,40 @@ +if __name__ == "__main__": + import sys + sys.path.insert(0,'..') +from restscrape.browser import browser as browser_class +from restscrape.scraper import scraper +from restscrape.proxy import create_proxy_iter +import time +US_PROXY_ITER = create_proxy_iter() + +def scrape(url,labels,max_tries=4,proxy_iter = None,wait_for = 0,raw_tags = True): + browser = browser_class(headless=False) + if proxy_iter is not None: + for trial in range(max_tries): + proxy_ip = next(proxy_iter) + try: + browser.restart_browser(start_page = url,proxy=proxy_ip) + if wait_for: + time.sleep(wait_for) + source = browser.get_source() + break + except Exception as e: + print(e) + print(proxy_ip) + proxy_iter.blacklist(proxy) + + else: + for trial in range(max_tries): + try: + if trial == 0: + browser.restart_browser(start_page = url) + else: + browser.open(url) + except Exception as e: + print(e) + + scraper = scraper(source) + return scraper.label_convert(raw_tags=True) + +if __name__ == "__main__": + ret = scrape('https://www.google.com',{'imgs':'//img'},wait_for = 10) \ No newline at end of file diff --git a/restscrape/browser.py b/restscrape/browser.py index d51881c..4fe59fa 100644 --- a/restscrape/browser.py +++ b/restscrape/browser.py @@ -22,7 +22,8 @@ class browser: if __name__ == "__main__": ext = os.path.join(os.getcwd(),'uBlock') else: - ext = os.path.join(os.path.dirname(__file__),'uBlock') + ext = os.path.abspath(os.path.join(os.path.dirname(__file__),'uBlock')) + print(ext) opts.setdefault('args',[]).extend(['--disable-extensions-except='+ext, '--load-extension='+ext]) if proxy is not None: opts.setdefault('args',[]).extend(['--proxy-server='+proxy]) diff --git a/restscrape/proxy.py b/restscrape/proxy.py index 1073775..0da65dc 100644 --- a/restscrape/proxy.py +++ b/restscrape/proxy.py @@ -26,7 +26,7 @@ class proxy_iter: def __len__(self): return len(self.proxy_set - self.bad) -def create_proxy_iter(url): +def create_proxy_iter(url = US_PROXY_URL): '''Create a proxy_iter from proxy_webpage''' resp = requests.get(url) resp.raise_for_status() diff --git a/restscrape/scraper.py b/restscrape/scraper.py index af63fa6..0503be7 100644 --- a/restscrape/scraper.py +++ b/restscrape/scraper.py @@ -17,6 +17,17 @@ class scraper: for row in table.xpath(rows_xpath)[0].findall('tr'): yield dict(zip(headers,(data.text for data in row.findall('td')))) + def label_convert(self,labels,raw_tags = False): + ret = {} + for label,xpath in labels.items(): + res = self.xpath(xpath) + if raw_tags: + ret[label] = list(etree.tostring(element, pretty_print=True) for element in res) + else: + ret[label] = list(element.text for element in res) + + return ret + def proxy_scraper(page_source): page = scraper(page_source) yield from page.extract_table(table="//table[@id='proxylisttable']",header_xpath="./thead/tr",rows_xpath="./tbody") diff --git a/restscrape/uBlock/1p-filters.html b/restscrape/uBlock/1p-filters.html new file mode 100644 index 0000000..9c482cf --- /dev/null +++ b/restscrape/uBlock/1p-filters.html @@ -0,0 +1,57 @@ + + + + + +uBlock — Your filters + + + + + + + + + + + + + +
+
+ +

+

+ + +    + + +

+
+
+ + + + + + + + + + + + + + + + + + + + + + + diff --git a/restscrape/uBlock/3p-filters.html b/restscrape/uBlock/3p-filters.html new file mode 100644 index 0000000..b35db0f --- /dev/null +++ b/restscrape/uBlock/3p-filters.html @@ -0,0 +1,69 @@ + + + + + +uBlock — Filter lists + + + + + + + + +
+ +
+ + + +
+ + + + + + + + + + + + + + diff --git a/restscrape/uBlock/about.html b/restscrape/uBlock/about.html new file mode 100644 index 0000000..081ddfc --- /dev/null +++ b/restscrape/uBlock/about.html @@ -0,0 +1,47 @@ + + + + + +uBlock — About + + + + + + +
+

+

Copyright (c) Raymond Hill 2014-present
+   +

+ +
+ + + + + + + + + + + diff --git a/restscrape/uBlock/asset-viewer.html b/restscrape/uBlock/asset-viewer.html new file mode 100644 index 0000000..8d02d2e --- /dev/null +++ b/restscrape/uBlock/asset-viewer.html @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + diff --git a/restscrape/uBlock/background.html b/restscrape/uBlock/background.html new file mode 100644 index 0000000..29b9a29 --- /dev/null +++ b/restscrape/uBlock/background.html @@ -0,0 +1,45 @@ + + + + +uBlock Origin + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/restscrape/uBlock/cloud-ui.html b/restscrape/uBlock/cloud-ui.html new file mode 100644 index 0000000..ed54676 --- /dev/null +++ b/restscrape/uBlock/cloud-ui.html @@ -0,0 +1,23 @@ + + + + + + + +
+ + + + +
+

+ +
+
+

+

+

+
+ + diff --git a/restscrape/uBlock/dashboard.html b/restscrape/uBlock/dashboard.html new file mode 100644 index 0000000..3e3106a --- /dev/null +++ b/restscrape/uBlock/dashboard.html @@ -0,0 +1,36 @@ + + + + + + + + + + + + +
+
+ +
+
+ + + + + + + + + + + + diff --git a/restscrape/uBlock/document-blocked.html b/restscrape/uBlock/document-blocked.html new file mode 100644 index 0000000..c9bc8d5 --- /dev/null +++ b/restscrape/uBlock/document-blocked.html @@ -0,0 +1,66 @@ + + + + + + + + + + + +
exclamation-triangle
+ +
+

+ +
+ +
+

+

+
+ +
+

+

+
+ +
+

+

+

+
+ + + + + + + + + + + + diff --git a/restscrape/uBlock/dyna-rules.html b/restscrape/uBlock/dyna-rules.html new file mode 100644 index 0000000..87bc10a --- /dev/null +++ b/restscrape/uBlock/dyna-rules.html @@ -0,0 +1,61 @@ + + + + + +uBlock — Dynamic filtering rules + + + + + + + + + + + + +
+
+

+
+
+
+

+ + +
+
+

+ + + +
+
 
+
+
+
+
+ + + + + + + + + + + + + + + + + + + diff --git a/restscrape/uBlock/epicker.html b/restscrape/uBlock/epicker.html new file mode 100644 index 0000000..a1c676d --- /dev/null +++ b/restscrape/uBlock/epicker.html @@ -0,0 +1,238 @@ + + +uBlock Origin Element Picker + + + + + + + diff --git a/restscrape/uBlock/is-webrtc-supported.html b/restscrape/uBlock/is-webrtc-supported.html new file mode 100644 index 0000000..d30b674 --- /dev/null +++ b/restscrape/uBlock/is-webrtc-supported.html @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/restscrape/uBlock/lib/codemirror/LICENSE b/restscrape/uBlock/lib/codemirror/LICENSE new file mode 100644 index 0000000..ff7db4b --- /dev/null +++ b/restscrape/uBlock/lib/codemirror/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (C) 2017 by Marijn Haverbeke and others + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/restscrape/uBlock/lib/codemirror/README.md b/restscrape/uBlock/lib/codemirror/README.md new file mode 100644 index 0000000..8bc6c93 --- /dev/null +++ b/restscrape/uBlock/lib/codemirror/README.md @@ -0,0 +1,49 @@ +# CodeMirror + +[![Build Status](https://travis-ci.org/codemirror/CodeMirror.svg)](https://travis-ci.org/codemirror/CodeMirror) +[![NPM version](https://img.shields.io/npm/v/codemirror.svg)](https://www.npmjs.org/package/codemirror) +[![Join the chat at https://gitter.im/codemirror/CodeMirror](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/codemirror/CodeMirror) +[Funding status: ![maintainer happiness](https://marijnhaverbeke.nl/fund/status_s.png?again)](https://marijnhaverbeke.nl/fund/) + +CodeMirror is a versatile text editor implemented in JavaScript for +the browser. It is specialized for editing code, and comes with over +100 language modes and various addons that implement more advanced +editing functionality. Every language comes with fully-featured code +and syntax highlighting to help with reading and editing complex code. + +A rich programming API and a CSS theming system are available for +customizing CodeMirror to fit your application, and extending it with +new functionality. + +You can find more information (and the +[manual](https://codemirror.net/doc/manual.html)) on the [project +page](https://codemirror.net). For questions and discussion, use the +[discussion forum](https://discuss.codemirror.net/). + +See +[CONTRIBUTING.md](https://github.com/codemirror/CodeMirror/blob/master/CONTRIBUTING.md) +for contributing guidelines. + +The CodeMirror community aims to be welcoming to everybody. We use the +[Contributor Covenant +(1.1)](http://contributor-covenant.org/version/1/1/0/) as our code of +conduct. + +### Installation + +Either get the [zip file](https://codemirror.net/codemirror.zip) with +the latest version, or make sure you have [Node](https://nodejs.org/) +installed and run: + + npm install codemirror + +**NOTE**: This is the source repository for the library, and not the +distribution channel. Cloning it is not the recommended way to install +the library, and will in fact not work unless you also run the build +step. + +### Quickstart + +To build the project, make sure you have Node.js installed (at least version 6) +and then `npm install`. To run, just open `index.html` in your +browser (you don't need to run a webserver). Run the tests with `npm test`. diff --git a/restscrape/uBlock/lib/codemirror/addon/display/panel.js b/restscrape/uBlock/lib/codemirror/addon/display/panel.js new file mode 100644 index 0000000..5faf1d5 --- /dev/null +++ b/restscrape/uBlock/lib/codemirror/addon/display/panel.js @@ -0,0 +1,127 @@ +// CodeMirror, copyright (c) by Marijn Haverbeke and others +// Distributed under an MIT license: https://codemirror.net/LICENSE + +(function(mod) { + if (typeof exports == "object" && typeof module == "object") // CommonJS + mod(require("../../lib/codemirror")); + else if (typeof define == "function" && define.amd) // AMD + define(["../../lib/codemirror"], mod); + else // Plain browser env + mod(CodeMirror); +})(function(CodeMirror) { + CodeMirror.defineExtension("addPanel", function(node, options) { + options = options || {}; + + if (!this.state.panels) initPanels(this); + + var info = this.state.panels; + var wrapper = info.wrapper; + var cmWrapper = this.getWrapperElement(); + var replace = options.replace instanceof Panel && !options.replace.cleared; + + if (options.after instanceof Panel && !options.after.cleared) { + wrapper.insertBefore(node, options.before.node.nextSibling); + } else if (options.before instanceof Panel && !options.before.cleared) { + wrapper.insertBefore(node, options.before.node); + } else if (replace) { + wrapper.insertBefore(node, options.replace.node); + info.panels++; + options.replace.clear(); + } else if (options.position == "bottom") { + wrapper.appendChild(node); + } else if (options.position == "before-bottom") { + wrapper.insertBefore(node, cmWrapper.nextSibling); + } else if (options.position == "after-top") { + wrapper.insertBefore(node, cmWrapper); + } else { + wrapper.insertBefore(node, wrapper.firstChild); + } + + var height = (options && options.height) || node.offsetHeight; + this._setSize(null, info.heightLeft -= height); + if (!replace) { + info.panels++; + } + if (options.stable && isAtTop(this, node)) + this.scrollTo(null, this.getScrollInfo().top + height) + + return new Panel(this, node, options, height); + }); + + function Panel(cm, node, options, height) { + this.cm = cm; + this.node = node; + this.options = options; + this.height = height; + this.cleared = false; + } + + Panel.prototype.clear = function() { + if (this.cleared) return; + this.cleared = true; + var info = this.cm.state.panels; + this.cm._setSize(null, info.heightLeft += this.height); + if (this.options.stable && isAtTop(this.cm, this.node)) + this.cm.scrollTo(null, this.cm.getScrollInfo().top - this.height) + info.wrapper.removeChild(this.node); + if (--info.panels == 0) removePanels(this.cm); + }; + + Panel.prototype.changed = function(height) { + var newHeight = height == null ? this.node.offsetHeight : height; + var info = this.cm.state.panels; + this.cm._setSize(null, info.heightLeft -= (newHeight - this.height)); + this.height = newHeight; + }; + + function initPanels(cm) { + var wrap = cm.getWrapperElement(); + var style = window.getComputedStyle ? window.getComputedStyle(wrap) : wrap.currentStyle; + var height = parseInt(style.height); + var info = cm.state.panels = { + setHeight: wrap.style.height, + heightLeft: height, + panels: 0, + wrapper: document.createElement("div") + }; + wrap.parentNode.insertBefore(info.wrapper, wrap); + var hasFocus = cm.hasFocus(); + info.wrapper.appendChild(wrap); + if (hasFocus) cm.focus(); + + cm._setSize = cm.setSize; + if (height != null) cm.setSize = function(width, newHeight) { + if (newHeight == null) return this._setSize(width, newHeight); + info.setHeight = newHeight; + if (typeof newHeight != "number") { + var px = /^(\d+\.?\d*)px$/.exec(newHeight); + if (px) { + newHeight = Number(px[1]); + } else { + info.wrapper.style.height = newHeight; + newHeight = info.wrapper.offsetHeight; + info.wrapper.style.height = ""; + } + } + cm._setSize(width, info.heightLeft += (newHeight - height)); + height = newHeight; + }; + } + + function removePanels(cm) { + var info = cm.state.panels; + cm.state.panels = null; + + var wrap = cm.getWrapperElement(); + info.wrapper.parentNode.replaceChild(wrap, info.wrapper); + wrap.style.height = info.setHeight; + cm.setSize = cm._setSize; + cm.setSize(); + } + + function isAtTop(cm, dom) { + for (var sibling = dom.nextSibling; sibling; sibling = sibling.nextSibling) + if (sibling == cm.getWrapperElement()) return true + return false + } +}); diff --git a/restscrape/uBlock/lib/codemirror/addon/merge/merge.css b/restscrape/uBlock/lib/codemirror/addon/merge/merge.css new file mode 100644 index 0000000..dadd7f5 --- /dev/null +++ b/restscrape/uBlock/lib/codemirror/addon/merge/merge.css @@ -0,0 +1,119 @@ +.CodeMirror-merge { + position: relative; + border: 1px solid #ddd; + white-space: pre; +} + +.CodeMirror-merge, .CodeMirror-merge .CodeMirror { + height: 350px; +} + +.CodeMirror-merge-2pane .CodeMirror-merge-pane { width: 47%; } +.CodeMirror-merge-2pane .CodeMirror-merge-gap { width: 6%; } +.CodeMirror-merge-3pane .CodeMirror-merge-pane { width: 31%; } +.CodeMirror-merge-3pane .CodeMirror-merge-gap { width: 3.5%; } + +.CodeMirror-merge-pane { + display: inline-block; + white-space: normal; + vertical-align: top; +} +.CodeMirror-merge-pane-rightmost { + position: absolute; + right: 0px; + z-index: 1; +} + +.CodeMirror-merge-gap { + z-index: 2; + display: inline-block; + height: 100%; + -moz-box-sizing: border-box; + box-sizing: border-box; + overflow: hidden; + border-left: 1px solid #ddd; + border-right: 1px solid #ddd; + position: relative; + background: #f8f8f8; +} + +.CodeMirror-merge-scrolllock-wrap { + position: absolute; + bottom: 0; left: 50%; +} +.CodeMirror-merge-scrolllock { + position: relative; + left: -50%; + cursor: pointer; + color: #555; + line-height: 1; +} +.CodeMirror-merge-scrolllock:after { + content: "\21db\00a0\00a0\21da"; +} +.CodeMirror-merge-scrolllock.CodeMirror-merge-scrolllock-enabled:after { + content: "\21db\21da"; +} + +.CodeMirror-merge-copybuttons-left, .CodeMirror-merge-copybuttons-right { + position: absolute; + left: 0; top: 0; + right: 0; bottom: 0; + line-height: 1; +} + +.CodeMirror-merge-copy { + position: absolute; + cursor: pointer; + color: #44c; + z-index: 3; +} + +.CodeMirror-merge-copy-reverse { + position: absolute; + cursor: pointer; + color: #44c; +} + +.CodeMirror-merge-copybuttons-left .CodeMirror-merge-copy { left: 2px; } +.CodeMirror-merge-copybuttons-right .CodeMirror-merge-copy { right: 2px; } + +.CodeMirror-merge-r-inserted, .CodeMirror-merge-l-inserted { + background-image: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAMAAAACCAYAAACddGYaAAAAGUlEQVQI12MwuCXy3+CWyH8GBgYGJgYkAABZbAQ9ELXurwAAAABJRU5ErkJggg==); + background-position: bottom left; + background-repeat: repeat-x; +} + +.CodeMirror-merge-r-deleted, .CodeMirror-merge-l-deleted { + background-image: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAMAAAACCAYAAACddGYaAAAAGUlEQVQI12M4Kyb2/6yY2H8GBgYGJgYkAABURgPz6Ks7wQAAAABJRU5ErkJggg==); + background-position: bottom left; + background-repeat: repeat-x; +} + +.CodeMirror-merge-r-chunk { background: #ffffe0; } +.CodeMirror-merge-r-chunk-start { border-top: 1px solid #ee8; } +.CodeMirror-merge-r-chunk-end { border-bottom: 1px solid #ee8; } +.CodeMirror-merge-r-connect { fill: #ffffe0; stroke: #ee8; stroke-width: 1px; } + +.CodeMirror-merge-l-chunk { background: #eef; } +.CodeMirror-merge-l-chunk-start { border-top: 1px solid #88e; } +.CodeMirror-merge-l-chunk-end { border-bottom: 1px solid #88e; } +.CodeMirror-merge-l-connect { fill: #eef; stroke: #88e; stroke-width: 1px; } + +.CodeMirror-merge-l-chunk.CodeMirror-merge-r-chunk { background: #dfd; } +.CodeMirror-merge-l-chunk-start.CodeMirror-merge-r-chunk-start { border-top: 1px solid #4e4; } +.CodeMirror-merge-l-chunk-end.CodeMirror-merge-r-chunk-end { border-bottom: 1px solid #4e4; } + +.CodeMirror-merge-collapsed-widget:before { + content: "(...)"; +} +.CodeMirror-merge-collapsed-widget { + cursor: pointer; + color: #88b; + background: #eef; + border: 1px solid #ddf; + font-size: 90%; + padding: 0 3px; + border-radius: 4px; +} +.CodeMirror-merge-collapsed-line .CodeMirror-gutter-elt { display: none; } diff --git a/restscrape/uBlock/lib/codemirror/addon/merge/merge.js b/restscrape/uBlock/lib/codemirror/addon/merge/merge.js new file mode 100644 index 0000000..63373f7 --- /dev/null +++ b/restscrape/uBlock/lib/codemirror/addon/merge/merge.js @@ -0,0 +1,1002 @@ +// CodeMirror, copyright (c) by Marijn Haverbeke and others +// Distributed under an MIT license: https://codemirror.net/LICENSE + +// declare global: diff_match_patch, DIFF_INSERT, DIFF_DELETE, DIFF_EQUAL + +(function(mod) { + if (typeof exports == "object" && typeof module == "object") // CommonJS + mod(require("../../lib/codemirror")); // Note non-packaged dependency diff_match_patch + else if (typeof define == "function" && define.amd) // AMD + define(["../../lib/codemirror", "diff_match_patch"], mod); + else // Plain browser env + mod(CodeMirror); +})(function(CodeMirror) { + "use strict"; + var Pos = CodeMirror.Pos; + var svgNS = "http://www.w3.org/2000/svg"; + + function DiffView(mv, type) { + this.mv = mv; + this.type = type; + this.classes = type == "left" + ? {chunk: "CodeMirror-merge-l-chunk", + start: "CodeMirror-merge-l-chunk-start", + end: "CodeMirror-merge-l-chunk-end", + insert: "CodeMirror-merge-l-inserted", + del: "CodeMirror-merge-l-deleted", + connect: "CodeMirror-merge-l-connect"} + : {chunk: "CodeMirror-merge-r-chunk", + start: "CodeMirror-merge-r-chunk-start", + end: "CodeMirror-merge-r-chunk-end", + insert: "CodeMirror-merge-r-inserted", + del: "CodeMirror-merge-r-deleted", + connect: "CodeMirror-merge-r-connect"}; + } + + DiffView.prototype = { + constructor: DiffView, + init: function(pane, orig, options) { + this.edit = this.mv.edit; + ;(this.edit.state.diffViews || (this.edit.state.diffViews = [])).push(this); + this.orig = CodeMirror(pane, copyObj({value: orig, readOnly: !this.mv.options.allowEditingOriginals}, copyObj(options))); + if (this.mv.options.connect == "align") { + if (!this.edit.state.trackAlignable) this.edit.state.trackAlignable = new TrackAlignable(this.edit) + this.orig.state.trackAlignable = new TrackAlignable(this.orig) + } + this.lockButton.title = this.edit.phrase("Toggle locked scrolling"); + + this.orig.state.diffViews = [this]; + var classLocation = options.chunkClassLocation || "background"; + if (Object.prototype.toString.call(classLocation) != "[object Array]") classLocation = [classLocation] + this.classes.classLocation = classLocation + + this.diff = getDiff(asString(orig), asString(options.value), this.mv.options.ignoreWhitespace); + this.chunks = getChunks(this.diff); + this.diffOutOfDate = this.dealigned = false; + this.needsScrollSync = null + + this.showDifferences = options.showDifferences !== false; + }, + registerEvents: function(otherDv) { + this.forceUpdate = registerUpdate(this); + setScrollLock(this, true, false); + registerScroll(this, otherDv); + }, + setShowDifferences: function(val) { + val = val !== false; + if (val != this.showDifferences) { + this.showDifferences = val; + this.forceUpdate("full"); + } + } + }; + + function ensureDiff(dv) { + if (dv.diffOutOfDate) { + dv.diff = getDiff(dv.orig.getValue(), dv.edit.getValue(), dv.mv.options.ignoreWhitespace); + dv.chunks = getChunks(dv.diff); + dv.diffOutOfDate = false; + CodeMirror.signal(dv.edit, "updateDiff", dv.diff); + } + } + + var updating = false; + function registerUpdate(dv) { + var edit = {from: 0, to: 0, marked: []}; + var orig = {from: 0, to: 0, marked: []}; + var debounceChange, updatingFast = false; + function update(mode) { + updating = true; + updatingFast = false; + if (mode == "full") { + if (dv.svg) clear(dv.svg); + if (dv.copyButtons) clear(dv.copyButtons); + clearMarks(dv.edit, edit.marked, dv.classes); + clearMarks(dv.orig, orig.marked, dv.classes); + edit.from = edit.to = orig.from = orig.to = 0; + } + ensureDiff(dv); + if (dv.showDifferences) { + updateMarks(dv.edit, dv.diff, edit, DIFF_INSERT, dv.classes); + updateMarks(dv.orig, dv.diff, orig, DIFF_DELETE, dv.classes); + } + + if (dv.mv.options.connect == "align") + alignChunks(dv); + makeConnections(dv); + if (dv.needsScrollSync != null) syncScroll(dv, dv.needsScrollSync) + + updating = false; + } + function setDealign(fast) { + if (updating) return; + dv.dealigned = true; + set(fast); + } + function set(fast) { + if (updating || updatingFast) return; + clearTimeout(debounceChange); + if (fast === true) updatingFast = true; + debounceChange = setTimeout(update, fast === true ? 20 : 250); + } + function change(_cm, change) { + if (!dv.diffOutOfDate) { + dv.diffOutOfDate = true; + edit.from = edit.to = orig.from = orig.to = 0; + } + // Update faster when a line was added/removed + setDealign(change.text.length - 1 != change.to.line - change.from.line); + } + function swapDoc() { + dv.diffOutOfDate = true; + dv.dealigned = true; + update("full"); + } + dv.edit.on("change", change); + dv.orig.on("change", change); + dv.edit.on("swapDoc", swapDoc); + dv.orig.on("swapDoc", swapDoc); + if (dv.mv.options.connect == "align") { + CodeMirror.on(dv.edit.state.trackAlignable, "realign", setDealign) + CodeMirror.on(dv.orig.state.trackAlignable, "realign", setDealign) + } + dv.edit.on("viewportChange", function() { set(false); }); + dv.orig.on("viewportChange", function() { set(false); }); + update(); + return update; + } + + function registerScroll(dv, otherDv) { + dv.edit.on("scroll", function() { + syncScroll(dv, true) && makeConnections(dv); + }); + dv.orig.on("scroll", function() { + syncScroll(dv, false) && makeConnections(dv); + if (otherDv) syncScroll(otherDv, true) && makeConnections(otherDv); + }); + } + + function syncScroll(dv, toOrig) { + // Change handler will do a refresh after a timeout when diff is out of date + if (dv.diffOutOfDate) { + if (dv.lockScroll && dv.needsScrollSync == null) dv.needsScrollSync = toOrig + return false + } + dv.needsScrollSync = null + if (!dv.lockScroll) return true; + var editor, other, now = +new Date; + if (toOrig) { editor = dv.edit; other = dv.orig; } + else { editor = dv.orig; other = dv.edit; } + // Don't take action if the position of this editor was recently set + // (to prevent feedback loops) + if (editor.state.scrollSetBy == dv && (editor.state.scrollSetAt || 0) + 250 > now) return false; + + var sInfo = editor.getScrollInfo(); + if (dv.mv.options.connect == "align") { + targetPos = sInfo.top; + } else { + var halfScreen = .5 * sInfo.clientHeight, midY = sInfo.top + halfScreen; + var mid = editor.lineAtHeight(midY, "local"); + var around = chunkBoundariesAround(dv.chunks, mid, toOrig); + var off = getOffsets(editor, toOrig ? around.edit : around.orig); + var offOther = getOffsets(other, toOrig ? around.orig : around.edit); + var ratio = (midY - off.top) / (off.bot - off.top); + var targetPos = (offOther.top - halfScreen) + ratio * (offOther.bot - offOther.top); + + var botDist, mix; + // Some careful tweaking to make sure no space is left out of view + // when scrolling to top or bottom. + if (targetPos > sInfo.top && (mix = sInfo.top / halfScreen) < 1) { + targetPos = targetPos * mix + sInfo.top * (1 - mix); + } else if ((botDist = sInfo.height - sInfo.clientHeight - sInfo.top) < halfScreen) { + var otherInfo = other.getScrollInfo(); + var botDistOther = otherInfo.height - otherInfo.clientHeight - targetPos; + if (botDistOther > botDist && (mix = botDist / halfScreen) < 1) + targetPos = targetPos * mix + (otherInfo.height - otherInfo.clientHeight - botDist) * (1 - mix); + } + } + + other.scrollTo(sInfo.left, targetPos); + other.state.scrollSetAt = now; + other.state.scrollSetBy = dv; + return true; + } + + function getOffsets(editor, around) { + var bot = around.after; + if (bot == null) bot = editor.lastLine() + 1; + return {top: editor.heightAtLine(around.before || 0, "local"), + bot: editor.heightAtLine(bot, "local")}; + } + + function setScrollLock(dv, val, action) { + dv.lockScroll = val; + if (val && action != false) syncScroll(dv, DIFF_INSERT) && makeConnections(dv); + (val ? CodeMirror.addClass : CodeMirror.rmClass)(dv.lockButton, "CodeMirror-merge-scrolllock-enabled"); + } + + // Updating the marks for editor content + + function removeClass(editor, line, classes) { + var locs = classes.classLocation + for (var i = 0; i < locs.length; i++) { + editor.removeLineClass(line, locs[i], classes.chunk); + editor.removeLineClass(line, locs[i], classes.start); + editor.removeLineClass(line, locs[i], classes.end); + } + } + + function clearMarks(editor, arr, classes) { + for (var i = 0; i < arr.length; ++i) { + var mark = arr[i]; + if (mark instanceof CodeMirror.TextMarker) + mark.clear(); + else if (mark.parent) + removeClass(editor, mark, classes); + } + arr.length = 0; + } + + // FIXME maybe add a margin around viewport to prevent too many updates + function updateMarks(editor, diff, state, type, classes) { + var vp = editor.getViewport(); + editor.operation(function() { + if (state.from == state.to || vp.from - state.to > 20 || state.from - vp.to > 20) { + clearMarks(editor, state.marked, classes); + markChanges(editor, diff, type, state.marked, vp.from, vp.to, classes); + state.from = vp.from; state.to = vp.to; + } else { + if (vp.from < state.from) { + markChanges(editor, diff, type, state.marked, vp.from, state.from, classes); + state.from = vp.from; + } + if (vp.to > state.to) { + markChanges(editor, diff, type, state.marked, state.to, vp.to, classes); + state.to = vp.to; + } + } + }); + } + + function addClass(editor, lineNr, classes, main, start, end) { + var locs = classes.classLocation, line = editor.getLineHandle(lineNr); + for (var i = 0; i < locs.length; i++) { + if (main) editor.addLineClass(line, locs[i], classes.chunk); + if (start) editor.addLineClass(line, locs[i], classes.start); + if (end) editor.addLineClass(line, locs[i], classes.end); + } + return line; + } + + function markChanges(editor, diff, type, marks, from, to, classes) { + var pos = Pos(0, 0); + var top = Pos(from, 0), bot = editor.clipPos(Pos(to - 1)); + var cls = type == DIFF_DELETE ? classes.del : classes.insert; + function markChunk(start, end) { + var bfrom = Math.max(from, start), bto = Math.min(to, end); + for (var i = bfrom; i < bto; ++i) + marks.push(addClass(editor, i, classes, true, i == start, i == end - 1)); + // When the chunk is empty, make sure a horizontal line shows up + if (start == end && bfrom == end && bto == end) { + if (bfrom) + marks.push(addClass(editor, bfrom - 1, classes, false, false, true)); + else + marks.push(addClass(editor, bfrom, classes, false, true, false)); + } + } + + var chunkStart = 0, pending = false; + for (var i = 0; i < diff.length; ++i) { + var part = diff[i], tp = part[0], str = part[1]; + if (tp == DIFF_EQUAL) { + var cleanFrom = pos.line + (startOfLineClean(diff, i) ? 0 : 1); + moveOver(pos, str); + var cleanTo = pos.line + (endOfLineClean(diff, i) ? 1 : 0); + if (cleanTo > cleanFrom) { + if (pending) { markChunk(chunkStart, cleanFrom); pending = false } + chunkStart = cleanTo; + } + } else { + pending = true + if (tp == type) { + var end = moveOver(pos, str, true); + var a = posMax(top, pos), b = posMin(bot, end); + if (!posEq(a, b)) + marks.push(editor.markText(a, b, {className: cls})); + pos = end; + } + } + } + if (pending) markChunk(chunkStart, pos.line + 1); + } + + // Updating the gap between editor and original + + function makeConnections(dv) { + if (!dv.showDifferences) return; + + if (dv.svg) { + clear(dv.svg); + var w = dv.gap.offsetWidth; + attrs(dv.svg, "width", w, "height", dv.gap.offsetHeight); + } + if (dv.copyButtons) clear(dv.copyButtons); + + var vpEdit = dv.edit.getViewport(), vpOrig = dv.orig.getViewport(); + var outerTop = dv.mv.wrap.getBoundingClientRect().top + var sTopEdit = outerTop - dv.edit.getScrollerElement().getBoundingClientRect().top + dv.edit.getScrollInfo().top + var sTopOrig = outerTop - dv.orig.getScrollerElement().getBoundingClientRect().top + dv.orig.getScrollInfo().top; + for (var i = 0; i < dv.chunks.length; i++) { + var ch = dv.chunks[i]; + if (ch.editFrom <= vpEdit.to && ch.editTo >= vpEdit.from && + ch.origFrom <= vpOrig.to && ch.origTo >= vpOrig.from) + drawConnectorsForChunk(dv, ch, sTopOrig, sTopEdit, w); + } + } + + function getMatchingOrigLine(editLine, chunks) { + var editStart = 0, origStart = 0; + for (var i = 0; i < chunks.length; i++) { + var chunk = chunks[i]; + if (chunk.editTo > editLine && chunk.editFrom <= editLine) return null; + if (chunk.editFrom > editLine) break; + editStart = chunk.editTo; + origStart = chunk.origTo; + } + return origStart + (editLine - editStart); + } + + // Combines information about chunks and widgets/markers to return + // an array of lines, in a single editor, that probably need to be + // aligned with their counterparts in the editor next to it. + function alignableFor(cm, chunks, isOrig) { + var tracker = cm.state.trackAlignable + var start = cm.firstLine(), trackI = 0 + var result = [] + for (var i = 0;; i++) { + var chunk = chunks[i] + var chunkStart = !chunk ? 1e9 : isOrig ? chunk.origFrom : chunk.editFrom + for (; trackI < tracker.alignable.length; trackI += 2) { + var n = tracker.alignable[trackI] + 1 + if (n <= start) continue + if (n <= chunkStart) result.push(n) + else break + } + if (!chunk) break + result.push(start = isOrig ? chunk.origTo : chunk.editTo) + } + return result + } + + // Given information about alignable lines in two editors, fill in + // the result (an array of three-element arrays) to reflect the + // lines that need to be aligned with each other. + function mergeAlignable(result, origAlignable, chunks, setIndex) { + var rI = 0, origI = 0, chunkI = 0, diff = 0 + outer: for (;; rI++) { + var nextR = result[rI], nextO = origAlignable[origI] + if (!nextR && nextO == null) break + + var rLine = nextR ? nextR[0] : 1e9, oLine = nextO == null ? 1e9 : nextO + while (chunkI < chunks.length) { + var chunk = chunks[chunkI] + if (chunk.origFrom <= oLine && chunk.origTo > oLine) { + origI++ + rI-- + continue outer; + } + if (chunk.editTo > rLine) { + if (chunk.editFrom <= rLine) continue outer; + break + } + diff += (chunk.origTo - chunk.origFrom) - (chunk.editTo - chunk.editFrom) + chunkI++ + } + if (rLine == oLine - diff) { + nextR[setIndex] = oLine + origI++ + } else if (rLine < oLine - diff) { + nextR[setIndex] = rLine + diff + } else { + var record = [oLine - diff, null, null] + record[setIndex] = oLine + result.splice(rI, 0, record) + origI++ + } + } + } + + function findAlignedLines(dv, other) { + var alignable = alignableFor(dv.edit, dv.chunks, false), result = [] + if (other) for (var i = 0, j = 0; i < other.chunks.length; i++) { + var n = other.chunks[i].editTo + while (j < alignable.length && alignable[j] < n) j++ + if (j == alignable.length || alignable[j] != n) alignable.splice(j++, 0, n) + } + for (var i = 0; i < alignable.length; i++) + result.push([alignable[i], null, null]) + + mergeAlignable(result, alignableFor(dv.orig, dv.chunks, true), dv.chunks, 1) + if (other) + mergeAlignable(result, alignableFor(other.orig, other.chunks, true), other.chunks, 2) + + return result + } + + function alignChunks(dv, force) { + if (!dv.dealigned && !force) return; + if (!dv.orig.curOp) return dv.orig.operation(function() { + alignChunks(dv, force); + }); + + dv.dealigned = false; + var other = dv.mv.left == dv ? dv.mv.right : dv.mv.left; + if (other) { + ensureDiff(other); + other.dealigned = false; + } + var linesToAlign = findAlignedLines(dv, other); + + // Clear old aligners + var aligners = dv.mv.aligners; + for (var i = 0; i < aligners.length; i++) + aligners[i].clear(); + aligners.length = 0; + + var cm = [dv.edit, dv.orig], scroll = []; + if (other) cm.push(other.orig); + for (var i = 0; i < cm.length; i++) + scroll.push(cm[i].getScrollInfo().top); + + for (var ln = 0; ln < linesToAlign.length; ln++) + alignLines(cm, linesToAlign[ln], aligners); + + for (var i = 0; i < cm.length; i++) + cm[i].scrollTo(null, scroll[i]); + } + + function alignLines(cm, lines, aligners) { + var maxOffset = 0, offset = []; + for (var i = 0; i < cm.length; i++) if (lines[i] != null) { + var off = cm[i].heightAtLine(lines[i], "local"); + offset[i] = off; + maxOffset = Math.max(maxOffset, off); + } + for (var i = 0; i < cm.length; i++) if (lines[i] != null) { + var diff = maxOffset - offset[i]; + if (diff > 1) + aligners.push(padAbove(cm[i], lines[i], diff)); + } + } + + function padAbove(cm, line, size) { + var above = true; + if (line > cm.lastLine()) { + line--; + above = false; + } + var elt = document.createElement("div"); + elt.className = "CodeMirror-merge-spacer"; + elt.style.height = size + "px"; elt.style.minWidth = "1px"; + return cm.addLineWidget(line, elt, {height: size, above: above, mergeSpacer: true, handleMouseEvents: true}); + } + + function drawConnectorsForChunk(dv, chunk, sTopOrig, sTopEdit, w) { + var flip = dv.type == "left"; + var top = dv.orig.heightAtLine(chunk.origFrom, "local", true) - sTopOrig; + if (dv.svg) { + var topLpx = top; + var topRpx = dv.edit.heightAtLine(chunk.editFrom, "local", true) - sTopEdit; + if (flip) { var tmp = topLpx; topLpx = topRpx; topRpx = tmp; } + var botLpx = dv.orig.heightAtLine(chunk.origTo, "local", true) - sTopOrig; + var botRpx = dv.edit.heightAtLine(chunk.editTo, "local", true) - sTopEdit; + if (flip) { var tmp = botLpx; botLpx = botRpx; botRpx = tmp; } + var curveTop = " C " + w/2 + " " + topRpx + " " + w/2 + " " + topLpx + " " + (w + 2) + " " + topLpx; + var curveBot = " C " + w/2 + " " + botLpx + " " + w/2 + " " + botRpx + " -1 " + botRpx; + attrs(dv.svg.appendChild(document.createElementNS(svgNS, "path")), + "d", "M -1 " + topRpx + curveTop + " L " + (w + 2) + " " + botLpx + curveBot + " z", + "class", dv.classes.connect); + } + if (dv.copyButtons) { + var copy = dv.copyButtons.appendChild(elt("div", dv.type == "left" ? "\u21dd" : "\u21dc", + "CodeMirror-merge-copy")); + var editOriginals = dv.mv.options.allowEditingOriginals; + copy.title = dv.edit.phrase(editOriginals ? "Push to left" : "Revert chunk"); + copy.chunk = chunk; + copy.style.top = (chunk.origTo > chunk.origFrom ? top : dv.edit.heightAtLine(chunk.editFrom, "local") - sTopEdit) + "px"; + + if (editOriginals) { + var topReverse = dv.edit.heightAtLine(chunk.editFrom, "local") - sTopEdit; + var copyReverse = dv.copyButtons.appendChild(elt("div", dv.type == "right" ? "\u21dd" : "\u21dc", + "CodeMirror-merge-copy-reverse")); + copyReverse.title = "Push to right"; + copyReverse.chunk = {editFrom: chunk.origFrom, editTo: chunk.origTo, + origFrom: chunk.editFrom, origTo: chunk.editTo}; + copyReverse.style.top = topReverse + "px"; + dv.type == "right" ? copyReverse.style.left = "2px" : copyReverse.style.right = "2px"; + } + } + } + + function copyChunk(dv, to, from, chunk) { + if (dv.diffOutOfDate) return; + var origStart = chunk.origTo > from.lastLine() ? Pos(chunk.origFrom - 1) : Pos(chunk.origFrom, 0) + var origEnd = Pos(chunk.origTo, 0) + var editStart = chunk.editTo > to.lastLine() ? Pos(chunk.editFrom - 1) : Pos(chunk.editFrom, 0) + var editEnd = Pos(chunk.editTo, 0) + var handler = dv.mv.options.revertChunk + if (handler) + handler(dv.mv, from, origStart, origEnd, to, editStart, editEnd) + else + to.replaceRange(from.getRange(origStart, origEnd), editStart, editEnd) + } + + // Merge view, containing 0, 1, or 2 diff views. + + var MergeView = CodeMirror.MergeView = function(node, options) { + if (!(this instanceof MergeView)) return new MergeView(node, options); + + this.options = options; + var origLeft = options.origLeft, origRight = options.origRight == null ? options.orig : options.origRight; + + var hasLeft = origLeft != null, hasRight = origRight != null; + var panes = 1 + (hasLeft ? 1 : 0) + (hasRight ? 1 : 0); + var wrap = [], left = this.left = null, right = this.right = null; + var self = this; + + if (hasLeft) { + left = this.left = new DiffView(this, "left"); + var leftPane = elt("div", null, "CodeMirror-merge-pane CodeMirror-merge-left"); + wrap.push(leftPane); + wrap.push(buildGap(left)); + } + + var editPane = elt("div", null, "CodeMirror-merge-pane CodeMirror-merge-editor"); + wrap.push(editPane); + + if (hasRight) { + right = this.right = new DiffView(this, "right"); + wrap.push(buildGap(right)); + var rightPane = elt("div", null, "CodeMirror-merge-pane CodeMirror-merge-right"); + wrap.push(rightPane); + } + + (hasRight ? rightPane : editPane).className += " CodeMirror-merge-pane-rightmost"; + + wrap.push(elt("div", null, null, "height: 0; clear: both;")); + + var wrapElt = this.wrap = node.appendChild(elt("div", wrap, "CodeMirror-merge CodeMirror-merge-" + panes + "pane")); + this.edit = CodeMirror(editPane, copyObj(options)); + + if (left) left.init(leftPane, origLeft, options); + if (right) right.init(rightPane, origRight, options); + if (options.collapseIdentical) + this.editor().operation(function() { + collapseIdenticalStretches(self, options.collapseIdentical); + }); + if (options.connect == "align") { + this.aligners = []; + alignChunks(this.left || this.right, true); + } + if (left) left.registerEvents(right) + if (right) right.registerEvents(left) + + + var onResize = function() { + if (left) makeConnections(left); + if (right) makeConnections(right); + }; + CodeMirror.on(window, "resize", onResize); + var resizeInterval = setInterval(function() { + for (var p = wrapElt.parentNode; p && p != document.body; p = p.parentNode) {} + if (!p) { clearInterval(resizeInterval); CodeMirror.off(window, "resize", onResize); } + }, 5000); + }; + + function buildGap(dv) { + var lock = dv.lockButton = elt("div", null, "CodeMirror-merge-scrolllock"); + var lockWrap = elt("div", [lock], "CodeMirror-merge-scrolllock-wrap"); + CodeMirror.on(lock, "click", function() { setScrollLock(dv, !dv.lockScroll); }); + var gapElts = [lockWrap]; + if (dv.mv.options.revertButtons !== false) { + dv.copyButtons = elt("div", null, "CodeMirror-merge-copybuttons-" + dv.type); + CodeMirror.on(dv.copyButtons, "click", function(e) { + var node = e.target || e.srcElement; + if (!node.chunk) return; + if (node.className == "CodeMirror-merge-copy-reverse") { + copyChunk(dv, dv.orig, dv.edit, node.chunk); + return; + } + copyChunk(dv, dv.edit, dv.orig, node.chunk); + }); + gapElts.unshift(dv.copyButtons); + } + if (dv.mv.options.connect != "align") { + var svg = document.createElementNS && document.createElementNS(svgNS, "svg"); + if (svg && !svg.createSVGRect) svg = null; + dv.svg = svg; + if (svg) gapElts.push(svg); + } + + return dv.gap = elt("div", gapElts, "CodeMirror-merge-gap"); + } + + MergeView.prototype = { + constructor: MergeView, + editor: function() { return this.edit; }, + rightOriginal: function() { return this.right && this.right.orig; }, + leftOriginal: function() { return this.left && this.left.orig; }, + setShowDifferences: function(val) { + if (this.right) this.right.setShowDifferences(val); + if (this.left) this.left.setShowDifferences(val); + }, + rightChunks: function() { + if (this.right) { ensureDiff(this.right); return this.right.chunks; } + }, + leftChunks: function() { + if (this.left) { ensureDiff(this.left); return this.left.chunks; } + } + }; + + function asString(obj) { + if (typeof obj == "string") return obj; + else return obj.getValue(); + } + + // Operations on diffs + var dmp; + function getDiff(a, b, ignoreWhitespace) { + if (!dmp) dmp = new diff_match_patch(); + + var diff = dmp.diff_main(a, b); + // The library sometimes leaves in empty parts, which confuse the algorithm + for (var i = 0; i < diff.length; ++i) { + var part = diff[i]; + if (ignoreWhitespace ? !/[^ \t]/.test(part[1]) : !part[1]) { + diff.splice(i--, 1); + } else if (i && diff[i - 1][0] == part[0]) { + diff.splice(i--, 1); + diff[i][1] += part[1]; + } + } + return diff; + } + + function getChunks(diff) { + var chunks = []; + if (!diff.length) return chunks; + var startEdit = 0, startOrig = 0; + var edit = Pos(0, 0), orig = Pos(0, 0); + for (var i = 0; i < diff.length; ++i) { + var part = diff[i], tp = part[0]; + if (tp == DIFF_EQUAL) { + var startOff = !startOfLineClean(diff, i) || edit.line < startEdit || orig.line < startOrig ? 1 : 0; + var cleanFromEdit = edit.line + startOff, cleanFromOrig = orig.line + startOff; + moveOver(edit, part[1], null, orig); + var endOff = endOfLineClean(diff, i) ? 1 : 0; + var cleanToEdit = edit.line + endOff, cleanToOrig = orig.line + endOff; + if (cleanToEdit > cleanFromEdit) { + if (i) chunks.push({origFrom: startOrig, origTo: cleanFromOrig, + editFrom: startEdit, editTo: cleanFromEdit}); + startEdit = cleanToEdit; startOrig = cleanToOrig; + } + } else { + moveOver(tp == DIFF_INSERT ? edit : orig, part[1]); + } + } + if (startEdit <= edit.line || startOrig <= orig.line) + chunks.push({origFrom: startOrig, origTo: orig.line + 1, + editFrom: startEdit, editTo: edit.line + 1}); + return chunks; + } + + function endOfLineClean(diff, i) { + if (i == diff.length - 1) return true; + var next = diff[i + 1][1]; + if ((next.length == 1 && i < diff.length - 2) || next.charCodeAt(0) != 10) return false; + if (i == diff.length - 2) return true; + next = diff[i + 2][1]; + return (next.length > 1 || i == diff.length - 3) && next.charCodeAt(0) == 10; + } + + function startOfLineClean(diff, i) { + if (i == 0) return true; + var last = diff[i - 1][1]; + if (last.charCodeAt(last.length - 1) != 10) return false; + if (i == 1) return true; + last = diff[i - 2][1]; + return last.charCodeAt(last.length - 1) == 10; + } + + function chunkBoundariesAround(chunks, n, nInEdit) { + var beforeE, afterE, beforeO, afterO; + for (var i = 0; i < chunks.length; i++) { + var chunk = chunks[i]; + var fromLocal = nInEdit ? chunk.editFrom : chunk.origFrom; + var toLocal = nInEdit ? chunk.editTo : chunk.origTo; + if (afterE == null) { + if (fromLocal > n) { afterE = chunk.editFrom; afterO = chunk.origFrom; } + else if (toLocal > n) { afterE = chunk.editTo; afterO = chunk.origTo; } + } + if (toLocal <= n) { beforeE = chunk.editTo; beforeO = chunk.origTo; } + else if (fromLocal <= n) { beforeE = chunk.editFrom; beforeO = chunk.origFrom; } + } + return {edit: {before: beforeE, after: afterE}, orig: {before: beforeO, after: afterO}}; + } + + function collapseSingle(cm, from, to) { + cm.addLineClass(from, "wrap", "CodeMirror-merge-collapsed-line"); + var widget = document.createElement("span"); + widget.className = "CodeMirror-merge-collapsed-widget"; + widget.title = cm.phrase("Identical text collapsed. Click to expand."); + var mark = cm.markText(Pos(from, 0), Pos(to - 1), { + inclusiveLeft: true, + inclusiveRight: true, + replacedWith: widget, + clearOnEnter: true + }); + function clear() { + mark.clear(); + cm.removeLineClass(from, "wrap", "CodeMirror-merge-collapsed-line"); + } + if (mark.explicitlyCleared) clear(); + CodeMirror.on(widget, "click", clear); + mark.on("clear", clear); + CodeMirror.on(widget, "click", clear); + return {mark: mark, clear: clear}; + } + + function collapseStretch(size, editors) { + var marks = []; + function clear() { + for (var i = 0; i < marks.length; i++) marks[i].clear(); + } + for (var i = 0; i < editors.length; i++) { + var editor = editors[i]; + var mark = collapseSingle(editor.cm, editor.line, editor.line + size); + marks.push(mark); + mark.mark.on("clear", clear); + } + return marks[0].mark; + } + + function unclearNearChunks(dv, margin, off, clear) { + for (var i = 0; i < dv.chunks.length; i++) { + var chunk = dv.chunks[i]; + for (var l = chunk.editFrom - margin; l < chunk.editTo + margin; l++) { + var pos = l + off; + if (pos >= 0 && pos < clear.length) clear[pos] = false; + } + } + } + + function collapseIdenticalStretches(mv, margin) { + if (typeof margin != "number") margin = 2; + var clear = [], edit = mv.editor(), off = edit.firstLine(); + for (var l = off, e = edit.lastLine(); l <= e; l++) clear.push(true); + if (mv.left) unclearNearChunks(mv.left, margin, off, clear); + if (mv.right) unclearNearChunks(mv.right, margin, off, clear); + + for (var i = 0; i < clear.length; i++) { + if (clear[i]) { + var line = i + off; + for (var size = 1; i < clear.length - 1 && clear[i + 1]; i++, size++) {} + if (size > margin) { + var editors = [{line: line, cm: edit}]; + if (mv.left) editors.push({line: getMatchingOrigLine(line, mv.left.chunks), cm: mv.left.orig}); + if (mv.right) editors.push({line: getMatchingOrigLine(line, mv.right.chunks), cm: mv.right.orig}); + var mark = collapseStretch(size, editors); + if (mv.options.onCollapse) mv.options.onCollapse(mv, line, size, mark); + } + } + } + } + + // General utilities + + function elt(tag, content, className, style) { + var e = document.createElement(tag); + if (className) e.className = className; + if (style) e.style.cssText = style; + if (typeof content == "string") e.appendChild(document.createTextNode(content)); + else if (content) for (var i = 0; i < content.length; ++i) e.appendChild(content[i]); + return e; + } + + function clear(node) { + for (var count = node.childNodes.length; count > 0; --count) + node.removeChild(node.firstChild); + } + + function attrs(elt) { + for (var i = 1; i < arguments.length; i += 2) + elt.setAttribute(arguments[i], arguments[i+1]); + } + + function copyObj(obj, target) { + if (!target) target = {}; + for (var prop in obj) if (obj.hasOwnProperty(prop)) target[prop] = obj[prop]; + return target; + } + + function moveOver(pos, str, copy, other) { + var out = copy ? Pos(pos.line, pos.ch) : pos, at = 0; + for (;;) { + var nl = str.indexOf("\n", at); + if (nl == -1) break; + ++out.line; + if (other) ++other.line; + at = nl + 1; + } + out.ch = (at ? 0 : out.ch) + (str.length - at); + if (other) other.ch = (at ? 0 : other.ch) + (str.length - at); + return out; + } + + // Tracks collapsed markers and line widgets, in order to be able to + // accurately align the content of two editors. + + var F_WIDGET = 1, F_WIDGET_BELOW = 2, F_MARKER = 4 + + function TrackAlignable(cm) { + this.cm = cm + this.alignable = [] + this.height = cm.doc.height + var self = this + cm.on("markerAdded", function(_, marker) { + if (!marker.collapsed) return + var found = marker.find(1) + if (found != null) self.set(found.line, F_MARKER) + }) + cm.on("markerCleared", function(_, marker, _min, max) { + if (max != null && marker.collapsed) + self.check(max, F_MARKER, self.hasMarker) + }) + cm.on("markerChanged", this.signal.bind(this)) + cm.on("lineWidgetAdded", function(_, widget, lineNo) { + if (widget.mergeSpacer) return + if (widget.above) self.set(lineNo - 1, F_WIDGET_BELOW) + else self.set(lineNo, F_WIDGET) + }) + cm.on("lineWidgetCleared", function(_, widget, lineNo) { + if (widget.mergeSpacer) return + if (widget.above) self.check(lineNo - 1, F_WIDGET_BELOW, self.hasWidgetBelow) + else self.check(lineNo, F_WIDGET, self.hasWidget) + }) + cm.on("lineWidgetChanged", this.signal.bind(this)) + cm.on("change", function(_, change) { + var start = change.from.line, nBefore = change.to.line - change.from.line + var nAfter = change.text.length - 1, end = start + nAfter + if (nBefore || nAfter) self.map(start, nBefore, nAfter) + self.check(end, F_MARKER, self.hasMarker) + if (nBefore || nAfter) self.check(change.from.line, F_MARKER, self.hasMarker) + }) + cm.on("viewportChange", function() { + if (self.cm.doc.height != self.height) self.signal() + }) + } + + TrackAlignable.prototype = { + signal: function() { + CodeMirror.signal(this, "realign") + this.height = this.cm.doc.height + }, + + set: function(n, flags) { + var pos = -1 + for (; pos < this.alignable.length; pos += 2) { + var diff = this.alignable[pos] - n + if (diff == 0) { + if ((this.alignable[pos + 1] & flags) == flags) return + this.alignable[pos + 1] |= flags + this.signal() + return + } + if (diff > 0) break + } + this.signal() + this.alignable.splice(pos, 0, n, flags) + }, + + find: function(n) { + for (var i = 0; i < this.alignable.length; i += 2) + if (this.alignable[i] == n) return i + return -1 + }, + + check: function(n, flag, pred) { + var found = this.find(n) + if (found == -1 || !(this.alignable[found + 1] & flag)) return + if (!pred.call(this, n)) { + this.signal() + var flags = this.alignable[found + 1] & ~flag + if (flags) this.alignable[found + 1] = flags + else this.alignable.splice(found, 2) + } + }, + + hasMarker: function(n) { + var handle = this.cm.getLineHandle(n) + if (handle.markedSpans) for (var i = 0; i < handle.markedSpans.length; i++) + if (handle.markedSpans[i].mark.collapsed && handle.markedSpans[i].to != null) + return true + return false + }, + + hasWidget: function(n) { + var handle = this.cm.getLineHandle(n) + if (handle.widgets) for (var i = 0; i < handle.widgets.length; i++) + if (!handle.widgets[i].above && !handle.widgets[i].mergeSpacer) return true + return false + }, + + hasWidgetBelow: function(n) { + if (n == this.cm.lastLine()) return false + var handle = this.cm.getLineHandle(n + 1) + if (handle.widgets) for (var i = 0; i < handle.widgets.length; i++) + if (handle.widgets[i].above && !handle.widgets[i].mergeSpacer) return true + return false + }, + + map: function(from, nBefore, nAfter) { + var diff = nAfter - nBefore, to = from + nBefore, widgetFrom = -1, widgetTo = -1 + for (var i = 0; i < this.alignable.length; i += 2) { + var n = this.alignable[i] + if (n == from && (this.alignable[i + 1] & F_WIDGET_BELOW)) widgetFrom = i + if (n == to && (this.alignable[i + 1] & F_WIDGET_BELOW)) widgetTo = i + if (n <= from) continue + else if (n < to) this.alignable.splice(i--, 2) + else this.alignable[i] += diff + } + if (widgetFrom > -1) { + var flags = this.alignable[widgetFrom + 1] + if (flags == F_WIDGET_BELOW) this.alignable.splice(widgetFrom, 2) + else this.alignable[widgetFrom + 1] = flags & ~F_WIDGET_BELOW + } + if (widgetTo > -1 && nAfter) + this.set(from + nAfter, F_WIDGET_BELOW) + } + } + + function posMin(a, b) { return (a.line - b.line || a.ch - b.ch) < 0 ? a : b; } + function posMax(a, b) { return (a.line - b.line || a.ch - b.ch) > 0 ? a : b; } + function posEq(a, b) { return a.line == b.line && a.ch == b.ch; } + + function findPrevDiff(chunks, start, isOrig) { + for (var i = chunks.length - 1; i >= 0; i--) { + var chunk = chunks[i]; + var to = (isOrig ? chunk.origTo : chunk.editTo) - 1; + if (to < start) return to; + } + } + + function findNextDiff(chunks, start, isOrig) { + for (var i = 0; i < chunks.length; i++) { + var chunk = chunks[i]; + var from = (isOrig ? chunk.origFrom : chunk.editFrom); + if (from > start) return from; + } + } + + function goNearbyDiff(cm, dir) { + var found = null, views = cm.state.diffViews, line = cm.getCursor().line; + if (views) for (var i = 0; i < views.length; i++) { + var dv = views[i], isOrig = cm == dv.orig; + ensureDiff(dv); + var pos = dir < 0 ? findPrevDiff(dv.chunks, line, isOrig) : findNextDiff(dv.chunks, line, isOrig); + if (pos != null && (found == null || (dir < 0 ? pos > found : pos < found))) + found = pos; + } + if (found != null) + cm.setCursor(found, 0); + else + return CodeMirror.Pass; + } + + CodeMirror.commands.goNextDiff = function(cm) { + return goNearbyDiff(cm, 1); + }; + CodeMirror.commands.goPrevDiff = function(cm) { + return goNearbyDiff(cm, -1); + }; +}); diff --git a/restscrape/uBlock/lib/codemirror/addon/scroll/annotatescrollbar.js b/restscrape/uBlock/lib/codemirror/addon/scroll/annotatescrollbar.js new file mode 100644 index 0000000..3566258 --- /dev/null +++ b/restscrape/uBlock/lib/codemirror/addon/scroll/annotatescrollbar.js @@ -0,0 +1,122 @@ +// CodeMirror, copyright (c) by Marijn Haverbeke and others +// Distributed under an MIT license: https://codemirror.net/LICENSE + +(function(mod) { + if (typeof exports == "object" && typeof module == "object") // CommonJS + mod(require("../../lib/codemirror")); + else if (typeof define == "function" && define.amd) // AMD + define(["../../lib/codemirror"], mod); + else // Plain browser env + mod(CodeMirror); +})(function(CodeMirror) { + "use strict"; + + CodeMirror.defineExtension("annotateScrollbar", function(options) { + if (typeof options == "string") options = {className: options}; + return new Annotation(this, options); + }); + + CodeMirror.defineOption("scrollButtonHeight", 0); + + function Annotation(cm, options) { + this.cm = cm; + this.options = options; + this.buttonHeight = options.scrollButtonHeight || cm.getOption("scrollButtonHeight"); + this.annotations = []; + this.doRedraw = this.doUpdate = null; + this.div = cm.getWrapperElement().appendChild(document.createElement("div")); + this.div.style.cssText = "position: absolute; right: 0; top: 0; z-index: 7; pointer-events: none"; + this.computeScale(); + + function scheduleRedraw(delay) { + clearTimeout(self.doRedraw); + self.doRedraw = setTimeout(function() { self.redraw(); }, delay); + } + + var self = this; + cm.on("refresh", this.resizeHandler = function() { + clearTimeout(self.doUpdate); + self.doUpdate = setTimeout(function() { + if (self.computeScale()) scheduleRedraw(20); + }, 100); + }); + cm.on("markerAdded", this.resizeHandler); + cm.on("markerCleared", this.resizeHandler); + if (options.listenForChanges !== false) + cm.on("change", this.changeHandler = function() { + scheduleRedraw(250); + }); + } + + Annotation.prototype.computeScale = function() { + var cm = this.cm; + var hScale = (cm.getWrapperElement().clientHeight - cm.display.barHeight - this.buttonHeight * 2) / + cm.getScrollerElement().scrollHeight + if (hScale != this.hScale) { + this.hScale = hScale; + return true; + } + }; + + Annotation.prototype.update = function(annotations) { + this.annotations = annotations; + this.redraw(); + }; + + Annotation.prototype.redraw = function(compute) { + if (compute !== false) this.computeScale(); + var cm = this.cm, hScale = this.hScale; + + var frag = document.createDocumentFragment(), anns = this.annotations; + + var wrapping = cm.getOption("lineWrapping"); + var singleLineH = wrapping && cm.defaultTextHeight() * 1.5; + var curLine = null, curLineObj = null; + function getY(pos, top) { + if (curLine != pos.line) { + curLine = pos.line; + curLineObj = cm.getLineHandle(curLine); + } + if ((curLineObj.widgets && curLineObj.widgets.length) || + (wrapping && curLineObj.height > singleLineH)) + return cm.charCoords(pos, "local")[top ? "top" : "bottom"]; + var topY = cm.heightAtLine(curLineObj, "local"); + return topY + (top ? 0 : curLineObj.height); + } + + var lastLine = cm.lastLine() + if (cm.display.barWidth) for (var i = 0, nextTop; i < anns.length; i++) { + var ann = anns[i]; + if (ann.to.line > lastLine) continue; + var top = nextTop || getY(ann.from, true) * hScale; + var bottom = getY(ann.to, false) * hScale; + while (i < anns.length - 1) { + if (anns[i + 1].to.line > lastLine) break; + nextTop = getY(anns[i + 1].from, true) * hScale; + if (nextTop > bottom + .9) break; + ann = anns[++i]; + bottom = getY(ann.to, false) * hScale; + } + if (bottom == top) continue; + var height = Math.max(bottom - top, 3); + + var elt = frag.appendChild(document.createElement("div")); + elt.style.cssText = "position: absolute; right: 0px; width: " + Math.max(cm.display.barWidth - 1, 2) + "px; top: " + + (top + this.buttonHeight) + "px; height: " + height + "px"; + elt.className = this.options.className; + if (ann.id) { + elt.setAttribute("annotation-id", ann.id); + } + } + this.div.textContent = ""; + this.div.appendChild(frag); + }; + + Annotation.prototype.clear = function() { + this.cm.off("refresh", this.resizeHandler); + this.cm.off("markerAdded", this.resizeHandler); + this.cm.off("markerCleared", this.resizeHandler); + if (this.changeHandler) this.cm.off("change", this.changeHandler); + this.div.parentNode.removeChild(this.div); + }; +}); diff --git a/restscrape/uBlock/lib/codemirror/addon/search/matchesonscrollbar.css b/restscrape/uBlock/lib/codemirror/addon/search/matchesonscrollbar.css new file mode 100644 index 0000000..77932cc --- /dev/null +++ b/restscrape/uBlock/lib/codemirror/addon/search/matchesonscrollbar.css @@ -0,0 +1,8 @@ +.CodeMirror-search-match { + background: gold; + border-top: 1px solid orange; + border-bottom: 1px solid orange; + -moz-box-sizing: border-box; + box-sizing: border-box; + opacity: .5; +} diff --git a/restscrape/uBlock/lib/codemirror/addon/search/matchesonscrollbar.js b/restscrape/uBlock/lib/codemirror/addon/search/matchesonscrollbar.js new file mode 100644 index 0000000..4645f5e --- /dev/null +++ b/restscrape/uBlock/lib/codemirror/addon/search/matchesonscrollbar.js @@ -0,0 +1,97 @@ +// CodeMirror, copyright (c) by Marijn Haverbeke and others +// Distributed under an MIT license: https://codemirror.net/LICENSE + +(function(mod) { + if (typeof exports == "object" && typeof module == "object") // CommonJS + mod(require("../../lib/codemirror"), require("./searchcursor"), require("../scroll/annotatescrollbar")); + else if (typeof define == "function" && define.amd) // AMD + define(["../../lib/codemirror", "./searchcursor", "../scroll/annotatescrollbar"], mod); + else // Plain browser env + mod(CodeMirror); +})(function(CodeMirror) { + "use strict"; + + CodeMirror.defineExtension("showMatchesOnScrollbar", function(query, caseFold, options) { + if (typeof options == "string") options = {className: options}; + if (!options) options = {}; + return new SearchAnnotation(this, query, caseFold, options); + }); + + function SearchAnnotation(cm, query, caseFold, options) { + this.cm = cm; + this.options = options; + var annotateOptions = {listenForChanges: false}; + for (var prop in options) annotateOptions[prop] = options[prop]; + if (!annotateOptions.className) annotateOptions.className = "CodeMirror-search-match"; + this.annotation = cm.annotateScrollbar(annotateOptions); + this.query = query; + this.caseFold = caseFold; + this.gap = {from: cm.firstLine(), to: cm.lastLine() + 1}; + this.matches = []; + this.update = null; + + this.findMatches(); + this.annotation.update(this.matches); + + var self = this; + cm.on("change", this.changeHandler = function(_cm, change) { self.onChange(change); }); + } + + var MAX_MATCHES = 1000; + + SearchAnnotation.prototype.findMatches = function() { + if (!this.gap) return; + for (var i = 0; i < this.matches.length; i++) { + var match = this.matches[i]; + if (match.from.line >= this.gap.to) break; + if (match.to.line >= this.gap.from) this.matches.splice(i--, 1); + } + var cursor = this.cm.getSearchCursor(this.query, CodeMirror.Pos(this.gap.from, 0), this.caseFold); + var maxMatches = this.options && this.options.maxMatches || MAX_MATCHES; + while (cursor.findNext()) { + var match = {from: cursor.from(), to: cursor.to()}; + if (match.from.line >= this.gap.to) break; + this.matches.splice(i++, 0, match); + if (this.matches.length > maxMatches) break; + } + this.gap = null; + }; + + function offsetLine(line, changeStart, sizeChange) { + if (line <= changeStart) return line; + return Math.max(changeStart, line + sizeChange); + } + + SearchAnnotation.prototype.onChange = function(change) { + var startLine = change.from.line; + var endLine = CodeMirror.changeEnd(change).line; + var sizeChange = endLine - change.to.line; + if (this.gap) { + this.gap.from = Math.min(offsetLine(this.gap.from, startLine, sizeChange), change.from.line); + this.gap.to = Math.max(offsetLine(this.gap.to, startLine, sizeChange), change.from.line); + } else { + this.gap = {from: change.from.line, to: endLine + 1}; + } + + if (sizeChange) for (var i = 0; i < this.matches.length; i++) { + var match = this.matches[i]; + var newFrom = offsetLine(match.from.line, startLine, sizeChange); + if (newFrom != match.from.line) match.from = CodeMirror.Pos(newFrom, match.from.ch); + var newTo = offsetLine(match.to.line, startLine, sizeChange); + if (newTo != match.to.line) match.to = CodeMirror.Pos(newTo, match.to.ch); + } + clearTimeout(this.update); + var self = this; + this.update = setTimeout(function() { self.updateAfterChange(); }, 250); + }; + + SearchAnnotation.prototype.updateAfterChange = function() { + this.findMatches(); + this.annotation.update(this.matches); + }; + + SearchAnnotation.prototype.clear = function() { + this.cm.off("change", this.changeHandler); + this.annotation.clear(); + }; +}); diff --git a/restscrape/uBlock/lib/codemirror/addon/search/searchcursor.js b/restscrape/uBlock/lib/codemirror/addon/search/searchcursor.js new file mode 100644 index 0000000..aae36df --- /dev/null +++ b/restscrape/uBlock/lib/codemirror/addon/search/searchcursor.js @@ -0,0 +1,293 @@ +// CodeMirror, copyright (c) by Marijn Haverbeke and others +// Distributed under an MIT license: https://codemirror.net/LICENSE + +(function(mod) { + if (typeof exports == "object" && typeof module == "object") // CommonJS + mod(require("../../lib/codemirror")) + else if (typeof define == "function" && define.amd) // AMD + define(["../../lib/codemirror"], mod) + else // Plain browser env + mod(CodeMirror) +})(function(CodeMirror) { + "use strict" + var Pos = CodeMirror.Pos + + function regexpFlags(regexp) { + var flags = regexp.flags + return flags != null ? flags : (regexp.ignoreCase ? "i" : "") + + (regexp.global ? "g" : "") + + (regexp.multiline ? "m" : "") + } + + function ensureFlags(regexp, flags) { + var current = regexpFlags(regexp), target = current + for (var i = 0; i < flags.length; i++) if (target.indexOf(flags.charAt(i)) == -1) + target += flags.charAt(i) + return current == target ? regexp : new RegExp(regexp.source, target) + } + + function maybeMultiline(regexp) { + return /\\s|\\n|\n|\\W|\\D|\[\^/.test(regexp.source) + } + + function searchRegexpForward(doc, regexp, start) { + regexp = ensureFlags(regexp, "g") + for (var line = start.line, ch = start.ch, last = doc.lastLine(); line <= last; line++, ch = 0) { + regexp.lastIndex = ch + var string = doc.getLine(line), match = regexp.exec(string) + if (match) + return {from: Pos(line, match.index), + to: Pos(line, match.index + match[0].length), + match: match} + } + } + + function searchRegexpForwardMultiline(doc, regexp, start) { + if (!maybeMultiline(regexp)) return searchRegexpForward(doc, regexp, start) + + regexp = ensureFlags(regexp, "gm") + var string, chunk = 1 + for (var line = start.line, last = doc.lastLine(); line <= last;) { + // This grows the search buffer in exponentially-sized chunks + // between matches, so that nearby matches are fast and don't + // require concatenating the whole document (in case we're + // searching for something that has tons of matches), but at the + // same time, the amount of retries is limited. + for (var i = 0; i < chunk; i++) { + if (line > last) break + var curLine = doc.getLine(line++) + string = string == null ? curLine : string + "\n" + curLine + } + chunk = chunk * 2 + regexp.lastIndex = start.ch + var match = regexp.exec(string) + if (match) { + var before = string.slice(0, match.index).split("\n"), inside = match[0].split("\n") + var startLine = start.line + before.length - 1, startCh = before[before.length - 1].length + return {from: Pos(startLine, startCh), + to: Pos(startLine + inside.length - 1, + inside.length == 1 ? startCh + inside[0].length : inside[inside.length - 1].length), + match: match} + } + } + } + + function lastMatchIn(string, regexp) { + var cutOff = 0, match + for (;;) { + regexp.lastIndex = cutOff + var newMatch = regexp.exec(string) + if (!newMatch) return match + match = newMatch + cutOff = match.index + (match[0].length || 1) + if (cutOff == string.length) return match + } + } + + function searchRegexpBackward(doc, regexp, start) { + regexp = ensureFlags(regexp, "g") + for (var line = start.line, ch = start.ch, first = doc.firstLine(); line >= first; line--, ch = -1) { + var string = doc.getLine(line) + if (ch > -1) string = string.slice(0, ch) + var match = lastMatchIn(string, regexp) + if (match) + return {from: Pos(line, match.index), + to: Pos(line, match.index + match[0].length), + match: match} + } + } + + function searchRegexpBackwardMultiline(doc, regexp, start) { + regexp = ensureFlags(regexp, "gm") + var string, chunk = 1 + for (var line = start.line, first = doc.firstLine(); line >= first;) { + for (var i = 0; i < chunk; i++) { + var curLine = doc.getLine(line--) + string = string == null ? curLine.slice(0, start.ch) : curLine + "\n" + string + } + chunk *= 2 + + var match = lastMatchIn(string, regexp) + if (match) { + var before = string.slice(0, match.index).split("\n"), inside = match[0].split("\n") + var startLine = line + before.length, startCh = before[before.length - 1].length + return {from: Pos(startLine, startCh), + to: Pos(startLine + inside.length - 1, + inside.length == 1 ? startCh + inside[0].length : inside[inside.length - 1].length), + match: match} + } + } + } + + var doFold, noFold + if (String.prototype.normalize) { + doFold = function(str) { return str.normalize("NFD").toLowerCase() } + noFold = function(str) { return str.normalize("NFD") } + } else { + doFold = function(str) { return str.toLowerCase() } + noFold = function(str) { return str } + } + + // Maps a position in a case-folded line back to a position in the original line + // (compensating for codepoints increasing in number during folding) + function adjustPos(orig, folded, pos, foldFunc) { + if (orig.length == folded.length) return pos + for (var min = 0, max = pos + Math.max(0, orig.length - folded.length);;) { + if (min == max) return min + var mid = (min + max) >> 1 + var len = foldFunc(orig.slice(0, mid)).length + if (len == pos) return mid + else if (len > pos) max = mid + else min = mid + 1 + } + } + + function searchStringForward(doc, query, start, caseFold) { + // Empty string would match anything and never progress, so we + // define it to match nothing instead. + if (!query.length) return null + var fold = caseFold ? doFold : noFold + var lines = fold(query).split(/\r|\n\r?/) + + search: for (var line = start.line, ch = start.ch, last = doc.lastLine() + 1 - lines.length; line <= last; line++, ch = 0) { + var orig = doc.getLine(line).slice(ch), string = fold(orig) + if (lines.length == 1) { + var found = string.indexOf(lines[0]) + if (found == -1) continue search + var start = adjustPos(orig, string, found, fold) + ch + return {from: Pos(line, adjustPos(orig, string, found, fold) + ch), + to: Pos(line, adjustPos(orig, string, found + lines[0].length, fold) + ch)} + } else { + var cutFrom = string.length - lines[0].length + if (string.slice(cutFrom) != lines[0]) continue search + for (var i = 1; i < lines.length - 1; i++) + if (fold(doc.getLine(line + i)) != lines[i]) continue search + var end = doc.getLine(line + lines.length - 1), endString = fold(end), lastLine = lines[lines.length - 1] + if (endString.slice(0, lastLine.length) != lastLine) continue search + return {from: Pos(line, adjustPos(orig, string, cutFrom, fold) + ch), + to: Pos(line + lines.length - 1, adjustPos(end, endString, lastLine.length, fold))} + } + } + } + + function searchStringBackward(doc, query, start, caseFold) { + if (!query.length) return null + var fold = caseFold ? doFold : noFold + var lines = fold(query).split(/\r|\n\r?/) + + search: for (var line = start.line, ch = start.ch, first = doc.firstLine() - 1 + lines.length; line >= first; line--, ch = -1) { + var orig = doc.getLine(line) + if (ch > -1) orig = orig.slice(0, ch) + var string = fold(orig) + if (lines.length == 1) { + var found = string.lastIndexOf(lines[0]) + if (found == -1) continue search + return {from: Pos(line, adjustPos(orig, string, found, fold)), + to: Pos(line, adjustPos(orig, string, found + lines[0].length, fold))} + } else { + var lastLine = lines[lines.length - 1] + if (string.slice(0, lastLine.length) != lastLine) continue search + for (var i = 1, start = line - lines.length + 1; i < lines.length - 1; i++) + if (fold(doc.getLine(start + i)) != lines[i]) continue search + var top = doc.getLine(line + 1 - lines.length), topString = fold(top) + if (topString.slice(topString.length - lines[0].length) != lines[0]) continue search + return {from: Pos(line + 1 - lines.length, adjustPos(top, topString, top.length - lines[0].length, fold)), + to: Pos(line, adjustPos(orig, string, lastLine.length, fold))} + } + } + } + + function SearchCursor(doc, query, pos, options) { + this.atOccurrence = false + this.doc = doc + pos = pos ? doc.clipPos(pos) : Pos(0, 0) + this.pos = {from: pos, to: pos} + + var caseFold + if (typeof options == "object") { + caseFold = options.caseFold + } else { // Backwards compat for when caseFold was the 4th argument + caseFold = options + options = null + } + + if (typeof query == "string") { + if (caseFold == null) caseFold = false + this.matches = function(reverse, pos) { + return (reverse ? searchStringBackward : searchStringForward)(doc, query, pos, caseFold) + } + } else { + query = ensureFlags(query, "gm") + if (!options || options.multiline !== false) + this.matches = function(reverse, pos) { + return (reverse ? searchRegexpBackwardMultiline : searchRegexpForwardMultiline)(doc, query, pos) + } + else + this.matches = function(reverse, pos) { + return (reverse ? searchRegexpBackward : searchRegexpForward)(doc, query, pos) + } + } + } + + SearchCursor.prototype = { + findNext: function() {return this.find(false)}, + findPrevious: function() {return this.find(true)}, + + find: function(reverse) { + var result = this.matches(reverse, this.doc.clipPos(reverse ? this.pos.from : this.pos.to)) + + // Implements weird auto-growing behavior on null-matches for + // backwards-compatiblity with the vim code (unfortunately) + while (result && CodeMirror.cmpPos(result.from, result.to) == 0) { + if (reverse) { + if (result.from.ch) result.from = Pos(result.from.line, result.from.ch - 1) + else if (result.from.line == this.doc.firstLine()) result = null + else result = this.matches(reverse, this.doc.clipPos(Pos(result.from.line - 1))) + } else { + if (result.to.ch < this.doc.getLine(result.to.line).length) result.to = Pos(result.to.line, result.to.ch + 1) + else if (result.to.line == this.doc.lastLine()) result = null + else result = this.matches(reverse, Pos(result.to.line + 1, 0)) + } + } + + if (result) { + this.pos = result + this.atOccurrence = true + return this.pos.match || true + } else { + var end = Pos(reverse ? this.doc.firstLine() : this.doc.lastLine() + 1, 0) + this.pos = {from: end, to: end} + return this.atOccurrence = false + } + }, + + from: function() {if (this.atOccurrence) return this.pos.from}, + to: function() {if (this.atOccurrence) return this.pos.to}, + + replace: function(newText, origin) { + if (!this.atOccurrence) return + var lines = CodeMirror.splitLines(newText) + this.doc.replaceRange(lines, this.pos.from, this.pos.to, origin) + this.pos.to = Pos(this.pos.from.line + lines.length - 1, + lines[lines.length - 1].length + (lines.length == 1 ? this.pos.from.ch : 0)) + } + } + + CodeMirror.defineExtension("getSearchCursor", function(query, pos, caseFold) { + return new SearchCursor(this.doc, query, pos, caseFold) + }) + CodeMirror.defineDocExtension("getSearchCursor", function(query, pos, caseFold) { + return new SearchCursor(this, query, pos, caseFold) + }) + + CodeMirror.defineExtension("selectMatches", function(query, caseFold) { + var ranges = [] + var cur = this.getSearchCursor(query, this.getCursor("from"), caseFold) + while (cur.findNext()) { + if (CodeMirror.cmpPos(cur.to(), this.getCursor("to")) > 0) break + ranges.push({anchor: cur.from(), head: cur.to()}) + } + if (ranges.length) + this.setSelections(ranges, 0) + }) +}); diff --git a/restscrape/uBlock/lib/codemirror/addon/selection/active-line.js b/restscrape/uBlock/lib/codemirror/addon/selection/active-line.js new file mode 100644 index 0000000..c7b14ce --- /dev/null +++ b/restscrape/uBlock/lib/codemirror/addon/selection/active-line.js @@ -0,0 +1,72 @@ +// CodeMirror, copyright (c) by Marijn Haverbeke and others +// Distributed under an MIT license: https://codemirror.net/LICENSE + +(function(mod) { + if (typeof exports == "object" && typeof module == "object") // CommonJS + mod(require("../../lib/codemirror")); + else if (typeof define == "function" && define.amd) // AMD + define(["../../lib/codemirror"], mod); + else // Plain browser env + mod(CodeMirror); +})(function(CodeMirror) { + "use strict"; + var WRAP_CLASS = "CodeMirror-activeline"; + var BACK_CLASS = "CodeMirror-activeline-background"; + var GUTT_CLASS = "CodeMirror-activeline-gutter"; + + CodeMirror.defineOption("styleActiveLine", false, function(cm, val, old) { + var prev = old == CodeMirror.Init ? false : old; + if (val == prev) return + if (prev) { + cm.off("beforeSelectionChange", selectionChange); + clearActiveLines(cm); + delete cm.state.activeLines; + } + if (val) { + cm.state.activeLines = []; + updateActiveLines(cm, cm.listSelections()); + cm.on("beforeSelectionChange", selectionChange); + } + }); + + function clearActiveLines(cm) { + for (var i = 0; i < cm.state.activeLines.length; i++) { + cm.removeLineClass(cm.state.activeLines[i], "wrap", WRAP_CLASS); + cm.removeLineClass(cm.state.activeLines[i], "background", BACK_CLASS); + cm.removeLineClass(cm.state.activeLines[i], "gutter", GUTT_CLASS); + } + } + + function sameArray(a, b) { + if (a.length != b.length) return false; + for (var i = 0; i < a.length; i++) + if (a[i] != b[i]) return false; + return true; + } + + function updateActiveLines(cm, ranges) { + var active = []; + for (var i = 0; i < ranges.length; i++) { + var range = ranges[i]; + var option = cm.getOption("styleActiveLine"); + if (typeof option == "object" && option.nonEmpty ? range.anchor.line != range.head.line : !range.empty()) + continue + var line = cm.getLineHandleVisualStart(range.head.line); + if (active[active.length - 1] != line) active.push(line); + } + if (sameArray(cm.state.activeLines, active)) return; + cm.operation(function() { + clearActiveLines(cm); + for (var i = 0; i < active.length; i++) { + cm.addLineClass(active[i], "wrap", WRAP_CLASS); + cm.addLineClass(active[i], "background", BACK_CLASS); + cm.addLineClass(active[i], "gutter", GUTT_CLASS); + } + cm.state.activeLines = active; + }); + } + + function selectionChange(cm, sel) { + updateActiveLines(cm, sel.ranges); + } +}); diff --git a/restscrape/uBlock/lib/diff/README.md b/restscrape/uBlock/lib/diff/README.md new file mode 100644 index 0000000..e1a90b0 --- /dev/null +++ b/restscrape/uBlock/lib/diff/README.md @@ -0,0 +1,34 @@ +# diff + +implementation of myers diff algorithm + +[![Build Status](https://travis-ci.org/Swatinem/diff.png?branch=master)](https://travis-ci.org/Swatinem/diff) +[![Coverage Status](https://coveralls.io/repos/Swatinem/diff/badge.png?branch=master)](https://coveralls.io/r/Swatinem/diff) +[![Dependency Status](https://gemnasium.com/Swatinem/diff.png)](https://gemnasium.com/Swatinem/diff) + + +This uses the [*An O(ND) Difference Algorithm and Its Variations*](http://www.xmailserver.org/diff2.pdf) +Also see http://simplygenius.net/Article/DiffTutorial2 and +http://www.mathertel.de/Diff/ViewSrc.aspx for more inspiration + +## Installation + + $ npm install diff + $ component install Swatinem/diff + +## Usage + +### diff(a, b, [eql(a, b)]) + +Given two arrays (or array-likes, such as strings) `a` and `b` and an optional +equal function `eql`, this will return an array with the following operations: +* *nop* the element is in both arrays +* *ins* the element is only in array `b` and will be inserted +* *del* the element in only in array `a` and will be removed +* *rep* the element from `a` will be replaced by the element from `b`. +This is essentially the same as a del+ins + +## License + + LGPLv3 + diff --git a/restscrape/uBlock/lib/diff/swatinem_diff.js b/restscrape/uBlock/lib/diff/swatinem_diff.js new file mode 100644 index 0000000..0601df0 --- /dev/null +++ b/restscrape/uBlock/lib/diff/swatinem_diff.js @@ -0,0 +1,243 @@ +/******************************************************************************* + + Key portions of code below was borrowed from: + https://github.com/Swatinem/diff + + License is LGPL3 (thanks!) as per: + https://github.com/Swatinem/diff/blob/b58391504759/README.md + + I chose to pick this implementation over + https://github.com/google/diff-match-patch as suggested by CodeMirror + because: + + - Code is clean and simple to read -- useful when unfamiliar with the diff + algorithm, this makes changing the code easier if/when needed. + + - Smaller -- diff_match_patch comes with an extended API most of which is + of no use to the current project. + - diff_match_patch uncompressed: 74.7 KB + - Swatinem's diff uncompressed: 3.66 KB + + - I can easily adapt Swatinem's diff to deal with arrays of strings, which + is best suited for the current project -- it natively work with arrays. + + I removed portions of code which are of no use for the current project. + + I modified the diff script generator (Diff.prototype.editscript) since I + need to generate a script which is compatible with the output of the + diff_match_patch, as expected by CodeMirror. + + 2018-12-20 gorhill: + There was an issue causing the wrong diff data to be issued, for instance + when diff-ing these two URLs on a character granularity basis (failure + point is marked): + | + /articles/5c1a7aae1854f30006cb26f7/lede/1545239527833-shutterstock_726 01757 2-copy.jpeg?crop=0.8889xw%3A0.9988xh%3B0.1089xw%2C0xh&resize=650%3A*&output-quality=55 + /articles/5c1a* 1854f30006cb2* /lede/15452* -shutterstock_* 017* 2-copy.jpeg?crop=0.* xw%3A* h%3B0.0* xw%2C0xh&resize=650%3A*&output-quality=55 + /articles/5c1aaea91854f30006cb2f1e/lede/1545253629235-shutterstock_106399017 2-copy.jpeg?crop=0.7749xw%3A1 xh%3B0.0391xw%2C0xh&resize=650%3A*&output-quality=55 + | + + Investigating, I found what appears to be the original source on which the + code below is based: + - "An O(ND) Difference Algorithm for C#" by Matthias Hertel + - http://www.mathertel.de/Diff/ViewSrc.aspx + - https://github.com/mathertel + + There was a difference; code had been commented out in the original source: + http://www.mathertel.de/Diff/DiffTest.aspx?oldfile=Diff.cs.v1&newfile=Diff.cs.v2 + + The developer noted: + > There have been overlapping boxes; that where analyzed partial differently. + > One return-point is enough. + + After applying the changes to the code below, the problematic diff-ing went + away: + | + /articles/5c1a7aae1854f30006cb26f7/lede/1545239527833-shutterstock_726 01757 2-copy.jpeg?crop=0.8889xw%3A0.9988xh%3B0.1089xw%2C0xh&resize=650%3A*&output-quality=55 + /articles/5c1a* 1854f30006cb2* /lede/15452* -shutterstock_* 017* 2-copy.jpeg?crop=0.* 9xw%3A* xh%3B0.* xw%2C0xh&resize=650%3A*&output-quality=55 + /articles/5c1aaea91854f30006cb2f1e/lede/1545253629235-shutterstock_106399017 2-copy.jpeg?crop=0.7749xw%3A1 xh%3B0.0391xw%2C0xh&resize=650%3A*&output-quality=55 + | + + So I will assume this was the issue. + + TODO: + - Apply other changes which were applied to the original code + +**/ + +'use strict'; + +(function(context) { + + // CodeMirror expect these globals: + context.DIFF_INSERT = 1; + context.DIFF_DELETE = -1; + context.DIFF_EQUAL = 0; + context.diff_match_patch = function(){}; + + context.diff_match_patch.prototype.diff_main = function(a, b) { + if ( a === b ) { return [ [ 0, a ] ]; } + var aa = a.match(/\n|[^\n]+\n?/g) || []; + var bb = b.match(/\n|[^\n]+\n?/g) || []; + var d = new Diff(aa, bb, eqlDefault); + return d.editscript(); + }; + + function eqlDefault(a, b) { return a === b; } + + function Diff(a, b, eql) { + this.a = a; + this.b = b; + this.eql = eql; + + this.moda = Array.apply(null, new Array(a.length)).map(true.valueOf, false); + this.modb = Array.apply(null, new Array(b.length)).map(true.valueOf, false); + + // just to save some allocations: + this.down = {}; + this.up = {}; + + this.lcs(0, a.length, 0, b.length); + } + + Diff.prototype.editscript = function Diff_editscript() { + var moda = this.moda, modb = this.modb; + var astart = 0, aend = moda.length; + var bstart = 0, bend = modb.length; + var result = []; + while (astart < aend || bstart < bend) { + if (astart < aend && bstart < bend) { + if (!moda[astart] && !modb[bstart]) { + result.push([ 0, this.a[astart] ]); + astart++; bstart++; + continue; + } else if (moda[astart] && modb[bstart]) { + result.push([ -1, this.a[astart] ]); + result.push([ 1, this.b[bstart] ]); + astart++; bstart++; + continue; + } + } + if (astart < aend && (bstart >= bend || moda[astart])) { + result.push([ -1, this.a[astart] ]); + astart++; + } + if (bstart < bend && (astart >= aend || modb[bstart])) { + result.push([ 1, this.b[bstart] ]); + bstart++; + } + } + return result; + }; + + Diff.prototype.lcs = function Diff_lcs(astart, aend, bstart, bend) { + var a = this.a, b = this.b, eql = this.eql; + // separate common head + while (astart < aend && bstart < bend && eql(a[astart], b[bstart])) { + astart++; bstart++; + } + // separate common tail + while (astart < aend && bstart < bend && eql(a[aend - 1], b[bend - 1])) { + aend--; bend--; + } + + if (astart === aend) { + // only insertions + while (bstart < bend) { + this.modb[bstart] = true; + bstart++; + } + } else if (bend === bstart) { + // only deletions + while (astart < aend) { + this.moda[astart] = true; + astart++; + } + } else { + var snake = this.snake(astart, aend, bstart, bend); + + this.lcs(astart, snake.x, bstart, snake.y); + this.lcs(snake.x, aend, snake.y, bend); + } + }; + + Diff.prototype.snake = function Diff_snake(astart, aend, bstart, bend) { + var a = this.a, b = this.b, eql = this.eql; + + var N = aend - astart, + M = bend - bstart; + + var kdown = astart - bstart; + var kup = aend - bend; + + var delta = N - M; + var deltaOdd = delta & 1; + + var down = this.down; + down[kdown + 1] = astart; + var up = this.up; + up[kup - 1] = aend; + + var Dmax = (N + M + 1) / 2; + for (var D = 0; D <= Dmax; D++) { + var k, x, y; + // forward path + for (k = kdown - D; k <= kdown + D; k += 2) { + if (k === kdown - D) { + x = down[k + 1]; // down + } else { + x = down[k - 1] + 1; // right + if ((k < kdown + D) && (down[k + 1] >= x)) { + x = down[k + 1]; // down + } + } + y = x - k; + + while (x < aend && y < bend && eql(a[x], b[y])) { + x++; y++; // diagonal + } + down[k] = x; + + if (deltaOdd && (kup - D < k) && (k < kup + D) && + up[k] <= down[k]) { + return { + x: down[k], + y: down[k] - k, + // u: up[k], + // v: up[k] - k, + }; + } + } + + // reverse path + for (k = kup - D; k <= kup + D; k += 2) { + if (k === kup + D) { + x = up[k - 1]; // up + } else { + x = up[k + 1] - 1; // left + if ((k > kup - D) && (up[k - 1] < x)) { + x = up[k - 1]; // up + } + } + y = x - k; + + while (x > astart && y > bstart && eql(a[x - 1], b[y - 1])) { + x--; y--; // diagonal + } + up[k] = x; + + if (!deltaOdd && (kdown - D <= k) && (k <= kdown + D) && + up[k] <= down[k]) { + return { + x: down[k], + y: down[k] - k, + // u: up[k], + // v: up[k] - k, + }; + } + } + } + }; + + return Diff; +})(self); diff --git a/restscrape/uBlock/lib/lz4/README.md b/restscrape/uBlock/lib/lz4/README.md new file mode 100644 index 0000000..2f6ba74 --- /dev/null +++ b/restscrape/uBlock/lib/lz4/README.md @@ -0,0 +1,52 @@ +## Purpose + +The purpose of this library is to implement LZ4 compression/decompression, +as documented at the official LZ4 repository: + +https://github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md + +The files in this directory are developed as a separate project at: + +https://github.com/gorhill/lz4-wasm + +## Files + +### `lz4-block-codec-any.js` + +The purpose is to instanciate a WebAssembly- or pure javascript-based +LZ4 block codec. + +If the choosen implementation is not specified, there will be an attempt to +create a WebAssembly-based instance. If for whatever reason this fails, a +pure javascript-based instance will be created. + +The script for either instance are dynamically loaded and only when needed, +such that no resources are wasted by keeping in memory code which won't be +used. + +### `lz4-block-codec-wasm.js` + +This contains the code to instanciate WebAssembly-based LZ4 block codec. Note +that the WebAssembly module is loaded using a `same-origin` fetch, hence +ensuring that no code outside the package is loaded. + +### `lz4-block-codec-js.js` + +This contains the code to instanciate pure javascript-based LZ4 block codec. + +This is used as a fallback implementation should WebAssembly not be available +for whatever reason. + +### `lz4-block-codec.wasm` + +This is the WebAssembly module, loaded by `lz4-block-codec-wasm.js` using a +`same-origin` fetch. + +### `lz4-block-codec.wat` + +The WebAssembly source code used to generate the WebAssembly module `lz4-block-codec.wasm`. + + wat2wasm ./lz4-block-codec.wat -o ./lz4-block-codec.wasm + wasm-opt ./lz4-block-codec.wasm -O4 -o ./lz4-block-codec.wasm + +You can get `wat2wasm` at , and `wasm-opt` at . diff --git a/restscrape/uBlock/lib/lz4/lz4-block-codec-any.js b/restscrape/uBlock/lib/lz4/lz4-block-codec-any.js new file mode 100644 index 0000000..ff00d9d --- /dev/null +++ b/restscrape/uBlock/lib/lz4/lz4-block-codec-any.js @@ -0,0 +1,171 @@ +/******************************************************************************* + + lz4-block-codec-any.js + A wrapper to instanciate a wasm- and/or js-based LZ4 block + encoder/decoder. + Copyright (C) 2018 Raymond Hill + + BSD-2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + Home: https://github.com/gorhill/lz4-wasm + + I used the same license as the one picked by creator of LZ4 out of respect + for his creation, see https://lz4.github.io/lz4/ + +*/ + +'use strict'; + +/******************************************************************************/ + +(function(context) { // >>>> Start of private namespace + +/******************************************************************************/ + +let wd = (function() { + let url = document.currentScript.src; + let match = /[^\/]+$/.exec(url); + return match !== null ? + url.slice(0, match.index) : + ''; +})(); + +let removeScript = function(script) { + if ( !script ) { return; } + if ( script.parentNode === null ) { return; } + script.parentNode.removeChild(script); +}; + +let createInstanceWASM = function() { + if ( context.LZ4BlockWASM instanceof Function ) { + let instance = new context.LZ4BlockWASM(); + return instance.init().then(( ) => { return instance; }); + } + if ( context.LZ4BlockWASM === null ) { + return Promise.resolve(null); + } + return new Promise((resolve, reject) => { + let script = document.createElement('script'); + script.src = wd + 'lz4-block-codec-wasm.js'; + script.addEventListener('load', ( ) => { + if ( context.LZ4BlockWASM instanceof Function === false ) { + context.LZ4BlockWASM = null; + context.LZ4BlockWASM = undefined; + resolve(null); + } else { + let instance = new context.LZ4BlockWASM(); + instance.init() + .then(( ) => { + resolve(instance); + }) + .catch(error => { + reject(error); + }); + } + }); + script.addEventListener('error', ( ) => { + context.LZ4BlockWASM = null; + resolve(null); + }); + document.head.appendChild(script); + removeScript(script); + }); +}; + +let createInstanceJS = function() { + if ( context.LZ4BlockJS instanceof Function ) { + let instance = new context.LZ4BlockJS(); + return instance.init().then(( ) => { return instance; }); + } + if ( context.LZ4BlockJS === null ) { + return Promise.resolve(null); + } + return new Promise((resolve, reject) => { + let script = document.createElement('script'); + script.src = wd + 'lz4-block-codec-js.js'; + script.addEventListener('load', ( ) => { + if ( context.LZ4BlockJS instanceof Function === false ) { + context.LZ4BlockJS = null; + resolve(null); + } else { + let instance = new context.LZ4BlockJS(); + instance.init() + .then(( ) => { + resolve(instance); + }) + .catch(error => { + reject(error); + }); + } + }); + script.addEventListener('error', ( ) => { + context.LZ4BlockJS = null; + resolve(null); + }); + document.head.appendChild(script); + removeScript(script); + }); +}; + +/******************************************************************************/ + +context.lz4BlockCodec = { + createInstance: function(flavor) { + let instantiator; + if ( flavor === 'wasm' ) { + instantiator = createInstanceWASM; + } else if ( flavor === 'js' ) { + instantiator = createInstanceJS; + } else { + instantiator = createInstanceWASM || createInstanceJS; + } + return (instantiator)() + .then(instance => { + if ( instance ) { return instance; } + if ( flavor === undefined ) { + return createInstanceJS(); + } + return null; + }) + .catch(( ) => { + if ( flavor === undefined ) { + return createInstanceJS(); + } + return null; + }); + }, + reset: function() { + context.LZ4BlockWASM = undefined; + context.LZ4BlockJS = undefined; + } +}; + +/******************************************************************************/ + +})(this || self); // <<<< End of private namespace + +/******************************************************************************/ diff --git a/restscrape/uBlock/lib/lz4/lz4-block-codec-js.js b/restscrape/uBlock/lib/lz4/lz4-block-codec-js.js new file mode 100644 index 0000000..ac0ebb6 --- /dev/null +++ b/restscrape/uBlock/lib/lz4/lz4-block-codec-js.js @@ -0,0 +1,297 @@ +/******************************************************************************* + + lz4-block-codec-js.js + A javascript wrapper around a pure javascript implementation of + LZ4 block format codec. + Copyright (C) 2018 Raymond Hill + + BSD-2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + Home: https://github.com/gorhill/lz4-wasm + + I used the same license as the one picked by creator of LZ4 out of respect + for his creation, see https://lz4.github.io/lz4/ + +*/ + +'use strict'; + +/******************************************************************************/ + +(function(context) { // >>>> Start of private namespace + +/******************************************************************************/ + +let growOutputBuffer = function(instance, size) { + if ( + instance.outputBuffer === undefined || + instance.outputBuffer.byteLength < size + ) { + instance.outputBuffer = new ArrayBuffer(size + 0xFFFF & 0x7FFF0000); + } + return instance.outputBuffer; +}; + +let encodeBound = function(size) { + return size > 0x7E000000 ? + 0 : + size + (size / 255 | 0) + 16; +}; + +let encodeBlock = function(instance, iBuf, oOffset) { + let iLen = iBuf.byteLength; + if ( iLen >= 0x7E000000 ) { throw new RangeError(); } + + // "The last match must start at least 12 bytes before end of block" + let lastMatchPos = iLen - 12; + + // "The last 5 bytes are always literals" + let lastLiteralPos = iLen - 5; + + if ( instance.hashTable === undefined ) { + instance.hashTable = new Int32Array(65536); + } + instance.hashTable.fill(-65536); + + if ( iBuf instanceof ArrayBuffer ) { + iBuf = new Uint8Array(iBuf); + } + + let oLen = oOffset + encodeBound(iLen); + let oBuf = new Uint8Array(growOutputBuffer(instance, oLen), 0, oLen); + let iPos = 0; + let oPos = oOffset; + let anchorPos = 0; + + // sequence-finding loop + for (;;) { + let refPos; + let mOffset; + let sequence = iBuf[iPos] << 8 | iBuf[iPos+1] << 16 | iBuf[iPos+2] << 24; + + // match-finding loop + while ( iPos <= lastMatchPos ) { + sequence = sequence >>> 8 | iBuf[iPos+3] << 24; + let hash = (sequence * 0x9E37 & 0xFFFF) + (sequence * 0x79B1 >>> 16) & 0xFFFF; + refPos = instance.hashTable[hash]; + instance.hashTable[hash] = iPos; + mOffset = iPos - refPos; + if ( + mOffset < 65536 && + iBuf[refPos+0] === ((sequence ) & 0xFF) && + iBuf[refPos+1] === ((sequence >>> 8) & 0xFF) && + iBuf[refPos+2] === ((sequence >>> 16) & 0xFF) && + iBuf[refPos+3] === ((sequence >>> 24) & 0xFF) + ) { + break; + } + iPos += 1; + } + + // no match found + if ( iPos > lastMatchPos ) { break; } + + // match found + let lLen = iPos - anchorPos; + let mLen = iPos; + iPos += 4; refPos += 4; + while ( iPos < lastLiteralPos && iBuf[iPos] === iBuf[refPos] ) { + iPos += 1; refPos += 1; + } + mLen = iPos - mLen; + let token = mLen < 19 ? mLen - 4 : 15; + + // write token, length of literals if needed + if ( lLen >= 15 ) { + oBuf[oPos++] = 0xF0 | token; + let l = lLen - 15; + while ( l >= 255 ) { + oBuf[oPos++] = 255; + l -= 255; + } + oBuf[oPos++] = l; + } else { + oBuf[oPos++] = (lLen << 4) | token; + } + + // write literals + while ( lLen-- ) { + oBuf[oPos++] = iBuf[anchorPos++]; + } + + if ( mLen === 0 ) { break; } + + // write offset of match + oBuf[oPos+0] = mOffset; + oBuf[oPos+1] = mOffset >>> 8; + oPos += 2; + + // write length of match if needed + if ( mLen >= 19 ) { + let l = mLen - 19; + while ( l >= 255 ) { + oBuf[oPos++] = 255; + l -= 255; + } + oBuf[oPos++] = l; + } + + anchorPos = iPos; + } + + // last sequence is literals only + let lLen = iLen - anchorPos; + if ( lLen >= 15 ) { + oBuf[oPos++] = 0xF0; + let l = lLen - 15; + while ( l >= 255 ) { + oBuf[oPos++] = 255; + l -= 255; + } + oBuf[oPos++] = l; + } else { + oBuf[oPos++] = lLen << 4; + } + while ( lLen-- ) { + oBuf[oPos++] = iBuf[anchorPos++]; + } + + return new Uint8Array(oBuf.buffer, 0, oPos); +}; + +let decodeBlock = function(instance, iBuf, iOffset, oLen) { + let iLen = iBuf.byteLength; + let oBuf = new Uint8Array(growOutputBuffer(instance, oLen), 0, oLen); + let iPos = iOffset, oPos = 0; + + while ( iPos < iLen ) { + let token = iBuf[iPos++]; + + // literals + let clen = token >>> 4; + + // length of literals + if ( clen !== 0 ) { + if ( clen === 15 ) { + let l; + for (;;) { + l = iBuf[iPos++]; + if ( l !== 255 ) { break; } + clen += 255; + } + clen += l; + } + + // copy literals + let end = iPos + clen; + while ( iPos < end ) { + oBuf[oPos++] = iBuf[iPos++]; + } + if ( iPos === iLen ) { break; } + } + + // match + let mOffset = iBuf[iPos+0] | (iBuf[iPos+1] << 8); + if ( mOffset === 0 || mOffset > oPos ) { return; } + iPos += 2; + + // length of match + clen = (token & 0x0F) + 4; + if ( clen === 19 ) { + let l; + for (;;) { + l = iBuf[iPos++]; + if ( l !== 255 ) { break; } + clen += 255; + } + clen += l; + } + + // copy match + let mPos = oPos - mOffset; + let end = oPos + clen; + while ( oPos < end ) { + oBuf[oPos++] = oBuf[mPos++]; + } + } + + return oBuf; +}; + +/******************************************************************************/ + +context.LZ4BlockJS = function() { + this.hashTable = undefined; + this.outputBuffer = undefined; +}; + +context.LZ4BlockJS.prototype = { + flavor: 'js', + init: function() { + return Promise.resolve(); + }, + + reset: function() { + this.hashTable = undefined; + this.outputBuffer = undefined; + }, + + bytesInUse: function() { + let bytesInUse = 0; + if ( this.hashTable !== undefined ) { + bytesInUse += this.hashTable.byteLength; + } + if ( this.outputBuffer !== undefined ) { + bytesInUse += this.outputBuffer.byteLength; + } + return bytesInUse; + }, + + encodeBlock: function(input, outputOffset) { + if ( input instanceof ArrayBuffer ) { + input = new Uint8Array(input); + } else if ( input instanceof Uint8Array === false ) { + throw new TypeError(); + } + return encodeBlock(this, input, outputOffset); + }, + + decodeBlock: function(input, inputOffset, outputSize) { + if ( input instanceof ArrayBuffer ) { + input = new Uint8Array(input); + } else if ( input instanceof Uint8Array === false ) { + throw new TypeError(); + } + return decodeBlock(this, input, inputOffset, outputSize); + } +}; + +/******************************************************************************/ + +})(this || self); // <<<< End of private namespace + +/******************************************************************************/ diff --git a/restscrape/uBlock/lib/lz4/lz4-block-codec-wasm.js b/restscrape/uBlock/lib/lz4/lz4-block-codec-wasm.js new file mode 100644 index 0000000..5a8e069 --- /dev/null +++ b/restscrape/uBlock/lib/lz4/lz4-block-codec-wasm.js @@ -0,0 +1,194 @@ +/******************************************************************************* + + lz4-block-codec-wasm.js + A javascript wrapper around a WebAssembly implementation of + LZ4 block format codec. + Copyright (C) 2018 Raymond Hill + + BSD-2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + Home: https://github.com/gorhill/lz4-wasm + + I used the same license as the one picked by creator of LZ4 out of respect + for his creation, see https://lz4.github.io/lz4/ + +*/ + +/* global WebAssembly */ + +'use strict'; + +/******************************************************************************/ + +(function(context) { // >>>> Start of private namespace + +/******************************************************************************/ + +let wd = (function() { + let url = document.currentScript.src; + let match = /[^\/]+$/.exec(url); + return match !== null ? + url.slice(0, match.index) : + ''; +})(); + +let growMemoryTo = function(wasmInstance, byteLength) { + let lz4api = wasmInstance.exports; + let neededByteLength = lz4api.getLinearMemoryOffset() + byteLength; + let pageCountBefore = lz4api.memory.buffer.byteLength >>> 16; + let pageCountAfter = (neededByteLength + 65535) >>> 16; + if ( pageCountAfter > pageCountBefore ) { + lz4api.memory.grow(pageCountAfter - pageCountBefore); + } + return lz4api.memory.buffer; +}; + +let encodeBlock = function(wasmInstance, inputArray, outputOffset) { + let lz4api = wasmInstance.exports; + let mem0 = lz4api.getLinearMemoryOffset(); + let hashTableSize = 65536 * 4; + let inputSize = inputArray.byteLength; + if ( inputSize >= 0x7E000000 ) { throw new RangeError(); } + let memSize = + hashTableSize + + inputSize + + outputOffset + lz4api.lz4BlockEncodeBound(inputSize); + let memBuffer = growMemoryTo(wasmInstance, memSize); + let hashTable = new Int32Array(memBuffer, mem0, 65536); + hashTable.fill(-65536, 0, 65536); + let inputMem = new Uint8Array(memBuffer, mem0 + hashTableSize, inputSize); + inputMem.set(inputArray); + let outputSize = lz4api.lz4BlockEncode( + mem0 + hashTableSize, + inputSize, + mem0 + hashTableSize + inputSize + outputOffset + ); + if ( outputSize === 0 ) { return; } + let outputArray = new Uint8Array( + memBuffer, + mem0 + hashTableSize + inputSize, + outputOffset + outputSize + ); + return outputArray; +}; + +let decodeBlock = function(wasmInstance, inputArray, inputOffset, outputSize) { + let inputSize = inputArray.byteLength; + let lz4api = wasmInstance.exports; + let mem0 = lz4api.getLinearMemoryOffset(); + let memSize = inputSize + outputSize; + let memBuffer = growMemoryTo(wasmInstance, memSize); + let inputArea = new Uint8Array(memBuffer, mem0, inputSize); + inputArea.set(inputArray); + outputSize = lz4api.lz4BlockDecode( + mem0 + inputOffset, + inputSize - inputOffset, + mem0 + inputSize + ); + if ( outputSize === 0 ) { return; } + return new Uint8Array(memBuffer, mem0 + inputSize, outputSize); +}; + +/******************************************************************************/ + +context.LZ4BlockWASM = function() { + this.lz4wasmInstance = undefined; +}; + +context.LZ4BlockWASM.prototype = { + flavor: 'wasm', + + init: function() { + if ( + typeof WebAssembly !== 'object' || + typeof WebAssembly.instantiateStreaming !== 'function' + ) { + this.lz4wasmInstance = null; + } + if ( this.lz4wasmInstance === null ) { + return Promise.reject(); + } + if ( this.lz4wasmInstance instanceof WebAssembly.Instance ) { + return Promise.resolve(this.lz4wasmInstance); + } + if ( this.lz4wasmInstance === undefined ) { + this.lz4wasmInstance = WebAssembly.instantiateStreaming( + fetch(wd + 'lz4-block-codec.wasm', { mode: 'same-origin' }) + ).then(result => { + this.lz4wasmInstance = undefined; + this.lz4wasmInstance = result && result.instance || null; + if ( this.lz4wasmInstance !== null ) { return this; } + return null; + }); + this.lz4wasmInstance.catch(( ) => { + this.lz4wasmInstance = null; + return null; + }); + } + return this.lz4wasmInstance; + }, + + reset: function() { + this.lz4wasmInstance = undefined; + }, + + bytesInUse: function() { + return this.lz4wasmInstance instanceof WebAssembly.Instance ? + this.lz4wasmInstance.exports.memory.buffer.byteLength : + 0; + }, + + encodeBlock: function(input, outputOffset) { + if ( this.lz4wasmInstance instanceof WebAssembly.Instance === false ) { + throw new Error('LZ4BlockWASM: not initialized'); + } + if ( input instanceof ArrayBuffer ) { + input = new Uint8Array(input); + } else if ( input instanceof Uint8Array === false ) { + throw new TypeError(); + } + return encodeBlock(this.lz4wasmInstance, input, outputOffset); + }, + + decodeBlock: function(input, inputOffset, outputSize) { + if ( this.lz4wasmInstance instanceof WebAssembly.Instance === false ) { + throw new Error('LZ4BlockWASM: not initialized'); + } + if ( input instanceof ArrayBuffer ) { + input = new Uint8Array(input); + } else if ( input instanceof Uint8Array === false ) { + throw new TypeError(); + } + return decodeBlock(this.lz4wasmInstance, input, inputOffset, outputSize); + } +}; + +/******************************************************************************/ + +})(this || self); // <<<< End of private namespace + +/******************************************************************************/ diff --git a/restscrape/uBlock/lib/lz4/lz4-block-codec.wasm b/restscrape/uBlock/lib/lz4/lz4-block-codec.wasm new file mode 100644 index 0000000..c57b079 Binary files /dev/null and b/restscrape/uBlock/lib/lz4/lz4-block-codec.wasm differ diff --git a/restscrape/uBlock/lib/lz4/lz4-block-codec.wat b/restscrape/uBlock/lib/lz4/lz4-block-codec.wat new file mode 100644 index 0000000..93e5dc1 --- /dev/null +++ b/restscrape/uBlock/lib/lz4/lz4-block-codec.wat @@ -0,0 +1,745 @@ +;; +;; lz4-block-codec.wat: a WebAssembly implementation of LZ4 block format codec +;; Copyright (C) 2018 Raymond Hill +;; +;; BSD-2-Clause License (http://www.opensource.org/licenses/bsd-license.php) +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are +;; met: +;; +;; 1. Redistributions of source code must retain the above copyright +;; notice, this list of conditions and the following disclaimer. +;; +;; 2. Redistributions in binary form must reproduce the above +;; copyright notice, this list of conditions and the following disclaimer +;; in the documentation and/or other materials provided with the +;; distribution. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +;; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +;; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +;; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +;; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; +;; Home: https://github.com/gorhill/lz4-wasm +;; +;; I used the same license as the one picked by creator of LZ4 out of respect +;; for his creation, see https://lz4.github.io/lz4/ +;; + +(module +;; +;; module start +;; + +;; (func $log (import "imports" "log") (param i32 i32 i32)) + +(memory (export "memory") 1) + +;; +;; Public functions +;; + +;; +;; Return an offset to the first byte of usable linear memory. +;; Might be useful in the future to reserve memory space for whatever purpose, +;; like config variables, etc. +;; +(func $getLinearMemoryOffset (export "getLinearMemoryOffset") + (result i32) + i32.const 0 +) + +;; +;; unsigned int lz4BlockEncodeBound() +;; +;; Return the maximum size of the output buffer holding the compressed data. +;; +;; Reference implementation: +;; https://github.com/lz4/lz4/blob/dev/lib/lz4.h#L156 +;; +(func (export "lz4BlockEncodeBound") + (param $ilen i32) + (result i32) + get_local $ilen + i32.const 0x7E000000 + i32.gt_u + if + i32.const 0 + return + end + get_local $ilen + get_local $ilen + i32.const 255 + i32.div_u + i32.add + i32.const 16 + i32.add +) + +;; +;; unsigned int lz4BlockEncode( +;; unsigned int inPtr, +;; unsigned int ilen, +;; unsigned int outPtr +;; ) +;; +;; https://github.com/lz4/lz4/blob/dev/lib/lz4.c#L651 +;; +;; The implementation below is modified from the reference one. +;; +;; - There is no skip adjustement for repeated failure to find a match. +;; +;; - All configurable values are hard-coded to match the generic version +;; of the compressor. +;; +;; Note the size of the input block is NOT encoded in the output buffer, it +;; is for the caller to figure how they will save that information on +;; their side. At this point it is probably a trivial amount of work to +;; implement the LZ4 frame format, which encode the content size, but this +;; is for another day. +;; +(func $lz4BlockEncode (export "lz4BlockEncode") + (param $inPtr i32) ;; pointer to start of input buffer + (param $ilen i32) ;; size of input buffer + (param $outPtr i32) ;; pointer to start of output buffer + (result i32) + (local $hashPtrBeg i32) ;; start of hash buffer + (local $hashPtr i32) ;; current hash entry + (local $anchorPtr i32) ;; anchor position in input + (local $inPtrEnd1 i32) ;; point in input at which match-finding must cease + (local $inPtrEnd2 i32) ;; point in input at which match-length finding must cease + (local $inPtrEnd i32) ;; point to end of input + (local $outPtrBeg i32) ;; start of output buffer + (local $refPtr i32) ;; start of match in input + (local $seq32 i32) ;; 4-byte value from current input position + (local $llen i32) ;; length of found literals + (local $moffset i32) ;; offset to found match from current input position + (local $mlen i32) ;; length of found match + get_local $ilen ;; empty input = empty output + i32.const 0x7E000000 ;; max input size: 0x7E000000 + i32.gt_u + if + i32.const 0 + return + end + get_local $ilen ;; "blocks < 13 bytes cannot be compressed" + i32.const 13 + i32.lt_u + if + i32.const 0 + return + end + call $getLinearMemoryOffset ;; hash table is at start of usable memory + set_local $hashPtrBeg + get_local $inPtr + tee_local $anchorPtr + get_local $ilen + i32.add + tee_local $inPtrEnd + i32.const -5 ;; "The last 5 bytes are always literals." + i32.add + tee_local $inPtrEnd2 + i32.const -7 ;; "The last match must start at least 12 bytes before end of block" + i32.add + set_local $inPtrEnd1 + get_local $outPtr + set_local $outPtrBeg + ;; + ;; sequence processing loop + ;; + block $noMoreSequence loop $nextSequence + get_local $inPtr + get_local $inPtrEnd1 + i32.ge_u ;; 5 or less bytes left? + br_if $noMoreSequence + get_local $inPtr ;; first sequence of 3 bytes before match-finding loop + i32.load8_u + i32.const 8 + i32.shl + get_local $inPtr + i32.load8_u offset=1 + i32.const 16 + i32.shl + i32.or + get_local $inPtr + i32.load8_u offset=2 + i32.const 24 + i32.shl + i32.or + set_local $seq32 + ;; + ;; match-finding loop + ;; + loop $findMatch block $noMatchFound + get_local $inPtr + get_local $inPtrEnd2 + i32.gt_u ;; less than 12 bytes left? + br_if $noMoreSequence + get_local $seq32 ;; update last byte of current sequence + i32.const 8 + i32.shr_u + get_local $inPtr + i32.load8_u offset=3 + i32.const 24 + i32.shl + i32.or + tee_local $seq32 + i32.const 0x9E3779B1 ;; compute 16-bit hash + i32.mul + i32.const 16 + i32.shr_u ;; hash value is at top of stack + i32.const 2 ;; lookup refPtr at hash entry + i32.shl + get_local $hashPtrBeg + i32.add + tee_local $hashPtr + i32.load + set_local $refPtr + get_local $hashPtr ;; update hash entry with inPtr + get_local $inPtr + i32.store + get_local $inPtr + get_local $refPtr + i32.sub + tee_local $moffset ;; remember match offset, we will need it in case of match + i32.const 0xFFFF + i32.gt_s ;; match offset > 65535 = unusable match + br_if $noMatchFound + ;; + ;; confirm match: different sequences can yield same hash + ;; compare-branch each byte to potentially save memory read ops + ;; + get_local $seq32 ;; byte 0 + i32.const 0xFF + i32.and + get_local $refPtr + i32.load8_u + i32.ne ;; refPtr[0] !== inPtr[0] + br_if $noMatchFound + get_local $seq32 ;; byte 1 + i32.const 8 + i32.shr_u + i32.const 0xFF + i32.and + get_local $refPtr + i32.load8_u offset=1 + i32.ne + br_if $noMatchFound ;; refPtr[1] !== inPtr[1] + get_local $seq32 ;; byte 2 + i32.const 16 + i32.shr_u + i32.const 0xFF + i32.and + get_local $refPtr + i32.load8_u offset=2 + i32.ne ;; refPtr[2] !== inPtr[2] + br_if $noMatchFound + get_local $seq32 ;; byte 3 + i32.const 24 + i32.shr_u + i32.const 0xFF + i32.and + get_local $refPtr + i32.load8_u offset=3 + i32.ne ;; refPtr[3] !== inPtr[3] + br_if $noMatchFound + ;; + ;; a valid match has been found at this point + ;; + get_local $inPtr ;; compute length of literals + get_local $anchorPtr + i32.sub + set_local $llen + get_local $inPtr ;; find match length + i32.const 4 ;; skip over confirmed 4-byte match + i32.add + set_local $inPtr + get_local $refPtr + i32.const 4 + i32.add + tee_local $mlen ;; remember refPtr to later compute match length + set_local $refPtr + block $endOfMatch loop ;; scan input buffer until match ends + get_local $inPtr + get_local $inPtrEnd2 + i32.ge_u + br_if $endOfMatch + get_local $inPtr + i32.load8_u + get_local $refPtr + i32.load8_u + i32.ne + br_if $endOfMatch + get_local $inPtr + i32.const 1 + i32.add + set_local $inPtr + get_local $refPtr + i32.const 1 + i32.add + set_local $refPtr + br 0 + end end $endOfMatch + ;; encode token + get_local $outPtr ;; output token + get_local $llen + get_local $refPtr + get_local $mlen + i32.sub + tee_local $mlen + call $writeToken + get_local $outPtr + i32.const 1 + i32.add + set_local $outPtr + get_local $llen ;; encode/write length of literals if needed + i32.const 15 + i32.ge_s + if + get_local $outPtr + get_local $llen + call $writeLength + set_local $outPtr + end + ;; copy literals + get_local $outPtr + get_local $anchorPtr + get_local $llen + call $copy + get_local $outPtr + get_local $llen + i32.add + set_local $outPtr + ;; encode match offset + get_local $outPtr + get_local $moffset + i32.store8 + get_local $outPtr + get_local $moffset + i32.const 8 + i32.shr_u + i32.store8 offset=1 + get_local $outPtr + i32.const 2 + i32.add + set_local $outPtr + get_local $mlen ;; encode/write length of match if needed + i32.const 15 + i32.ge_s + if + get_local $outPtr + get_local $mlen + call $writeLength + set_local $outPtr + end + get_local $inPtr ;; advance anchor to current position + set_local $anchorPtr + br $nextSequence + end $noMatchFound + get_local $inPtr ;; no match found: advance to next byte + i32.const 1 + i32.add + set_local $inPtr + br $findMatch end ;; match offset > 65535 = unusable match + end end $noMoreSequence + ;; + ;; generate last (match-less) sequence if compression succeeded + ;; + get_local $outPtr + get_local $outPtrBeg + i32.eq + if + i32.const 0 + return + end + get_local $outPtr + get_local $inPtrEnd + get_local $anchorPtr + i32.sub + tee_local $llen + i32.const 0 + call $writeToken + get_local $outPtr + i32.const 1 + i32.add + set_local $outPtr + get_local $llen + i32.const 15 + i32.ge_u + if + get_local $outPtr + get_local $llen + call $writeLength + set_local $outPtr + end + get_local $outPtr + get_local $anchorPtr + get_local $llen + call $copy + get_local $outPtr ;; return number of written bytes + get_local $llen + i32.add + get_local $outPtrBeg + i32.sub +) + +;; +;; unsigned int lz4BlockDecode( +;; unsigned int inPtr, +;; unsigned int ilen +;; unsigned int outPtr +;; ) +;; +;; Reference: +;; https://github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md +;; +(func (export "lz4BlockDecode") + (param $inPtr0 i32) ;; start of input buffer + (param $ilen i32) ;; length of input buffer + (param $outPtr0 i32) ;; start of output buffer + (result i32) + (local $inPtr i32) ;; current position in input buffer + (local $inPtrEnd i32) ;; end of input buffer + (local $outPtr i32) ;; current position in output buffer + (local $matchPtr i32) ;; position of current match + (local $token i32) ;; sequence token + (local $clen i32) ;; number of bytes to copy + (local $_ i32) ;; general purpose variable + get_local $ilen ;; if ( ilen == 0 ) { return 0; } + i32.eqz + if + i32.const 0 + return + end + get_local $inPtr0 + tee_local $inPtr ;; current position in input buffer + get_local $ilen + i32.add + set_local $inPtrEnd + get_local $outPtr0 ;; start of output buffer + set_local $outPtr ;; current position in output buffer + block $noMoreSequence loop ;; iterate through all sequences + get_local $inPtr + get_local $inPtrEnd + i32.ge_u + br_if $noMoreSequence ;; break when nothing left to read in input buffer + get_local $inPtr ;; read token -- consume one byte + i32.load8_u + get_local $inPtr + i32.const 1 + i32.add + set_local $inPtr + tee_local $token ;; extract length of literals from token + i32.const 4 + i32.shr_u + tee_local $clen ;; consume extra length bytes if present + i32.eqz + if else + get_local $clen + i32.const 15 + i32.eq + if loop + get_local $inPtr + i32.load8_u + get_local $inPtr + i32.const 1 + i32.add + set_local $inPtr + tee_local $_ + get_local $clen + i32.add + set_local $clen + get_local $_ + i32.const 255 + i32.eq + br_if 0 + end end + get_local $outPtr ;; copy literals to ouput buffer + get_local $inPtr + get_local $clen + call $copy + get_local $outPtr ;; advance output buffer pointer past copy + get_local $clen + i32.add + set_local $outPtr + get_local $clen ;; advance input buffer pointer past literals + get_local $inPtr + i32.add + tee_local $inPtr + get_local $inPtrEnd ;; exit if this is the last sequence + i32.eq + br_if $noMoreSequence + end + get_local $outPtr ;; read match offset + get_local $inPtr + i32.load8_u + get_local $inPtr + i32.load8_u offset=1 + i32.const 8 + i32.shl + i32.or + i32.sub + tee_local $matchPtr + get_local $outPtr ;; match position can't be outside input buffer bounds + i32.eq + br_if $noMoreSequence + get_local $matchPtr + get_local $inPtrEnd + i32.lt_u + br_if $noMoreSequence + get_local $inPtr ;; advance input pointer past match offset bytes + i32.const 2 + i32.add + set_local $inPtr + get_local $token ;; extract length of match from token + i32.const 15 + i32.and + i32.const 4 + i32.add + tee_local $clen + i32.const 19 ;; consume extra length bytes if present + i32.eq + if loop + get_local $inPtr + i32.load8_u + get_local $inPtr + i32.const 1 + i32.add + set_local $inPtr + tee_local $_ + get_local $clen + i32.add + set_local $clen + get_local $_ + i32.const 255 + i32.eq + br_if 0 + end end + get_local $outPtr ;; copy match to ouput buffer + get_local $matchPtr + get_local $clen + call $copy + get_local $clen ;; advance output buffer pointer past copy + get_local $outPtr + i32.add + set_local $outPtr + br 0 + end end $noMoreSequence + get_local $outPtr ;; return number of written bytes + get_local $outPtr0 + i32.sub +) + +;; +;; Private functions +;; + +;; +;; Encode a sequence token +;; +;; Reference documentation: +;; https://github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md +;; +(func $writeToken + (param $outPtr i32) + (param $llen i32) + (param $mlen i32) + get_local $outPtr + get_local $llen + i32.const 15 + get_local $llen + i32.const 15 + i32.lt_u + select + i32.const 4 + i32.shl + get_local $mlen + i32.const 15 + get_local $mlen + i32.const 15 + i32.lt_u + select + i32.or + i32.store8 +) + +;; +;; Encode and output length bytes. The return value is the pointer following +;; the last byte written. +;; +;; Reference documentation: +;; https://github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md +;; +(func $writeLength + (param $outPtr i32) + (param $len i32) + (result i32) + get_local $len + i32.const 15 + i32.sub + set_local $len + loop + get_local $outPtr + get_local $len + i32.const 255 + get_local $len + i32.const 255 + i32.lt_u + select + i32.store8 + get_local $outPtr + i32.const 1 + i32.add + set_local $outPtr + get_local $len + i32.const 255 + i32.sub + tee_local $len + i32.const 0 + i32.ge_s + br_if 0 + end + get_local $outPtr +) + +;; +;; Copy n bytes from source to destination. +;; +;; It is overlap-safe only from left-to-right -- which is only what is +;; required in the current module. +;; +(func $copy + (param $dst i32) + (param $src i32) + (param $len i32) + block $lessThan8 loop + get_local $len + i32.const 8 + i32.lt_u + br_if $lessThan8 + get_local $dst + get_local $src + i32.load8_u + i32.store8 + get_local $dst + get_local $src + i32.load8_u offset=1 + i32.store8 offset=1 + get_local $dst + get_local $src + i32.load8_u offset=2 + i32.store8 offset=2 + get_local $dst + get_local $src + i32.load8_u offset=3 + i32.store8 offset=3 + get_local $dst + get_local $src + i32.load8_u offset=4 + i32.store8 offset=4 + get_local $dst + get_local $src + i32.load8_u offset=5 + i32.store8 offset=5 + get_local $dst + get_local $src + i32.load8_u offset=6 + i32.store8 offset=6 + get_local $dst + get_local $src + i32.load8_u offset=7 + i32.store8 offset=7 + get_local $dst + i32.const 8 + i32.add + set_local $dst + get_local $src + i32.const 8 + i32.add + set_local $src + get_local $len + i32.const -8 + i32.add + set_local $len + br 0 + end end $lessThan8 + get_local $len + i32.const 4 + i32.ge_u + if + get_local $dst + get_local $src + i32.load8_u + i32.store8 + get_local $dst + get_local $src + i32.load8_u offset=1 + i32.store8 offset=1 + get_local $dst + get_local $src + i32.load8_u offset=2 + i32.store8 offset=2 + get_local $dst + get_local $src + i32.load8_u offset=3 + i32.store8 offset=3 + get_local $dst + i32.const 4 + i32.add + set_local $dst + get_local $src + i32.const 4 + i32.add + set_local $src + get_local $len + i32.const -4 + i32.add + set_local $len + end + get_local $len + i32.const 2 + i32.ge_u + if + get_local $dst + get_local $src + i32.load8_u + i32.store8 + get_local $dst + get_local $src + i32.load8_u offset=1 + i32.store8 offset=1 + get_local $dst + i32.const 2 + i32.add + set_local $dst + get_local $src + i32.const 2 + i32.add + set_local $src + get_local $len + i32.const -2 + i32.add + set_local $len + end + get_local $len + i32.eqz + if else + get_local $dst + get_local $src + i32.load8_u + i32.store8 + end +) + +;; +;; module end +;; +) diff --git a/restscrape/uBlock/lib/publicsuffixlist.js b/restscrape/uBlock/lib/publicsuffixlist.js new file mode 100644 index 0000000..cc7ae2d --- /dev/null +++ b/restscrape/uBlock/lib/publicsuffixlist.js @@ -0,0 +1,328 @@ +/******************************************************************************* + + publicsuffixlist.js - an efficient javascript implementation to deal with + Mozilla Foundation's Public Suffix List + Copyright (C) 2013-2018 Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + +*/ + +/*! Home: https://github.com/gorhill/publicsuffixlist.js */ + +/* + This code is mostly dumb: I consider this to be lower-level code, thus + in order to ensure efficiency, the caller is responsible for sanitizing + the inputs. +*/ + +/******************************************************************************/ + +// A single instance of PublicSuffixList is enough. + +;(function(root) { + +'use strict'; + +/******************************************************************************/ + +let exceptions = new Map(); +let rules = new Map(); + +// This value dictate how the search will be performed: +// < this.cutoffLength = indexOf() +// >= this.cutoffLength = binary search +const cutoffLength = 256; +const mustPunycode = /[^a-z0-9.-]/; + +/******************************************************************************/ + +// In the context of this code, a domain is defined as: +// "{label}.{public suffix}". +// A single standalone label is a public suffix as per +// http://publicsuffix.org/list/: +// "If no rules match, the prevailing rule is '*' " +// This means 'localhost' is not deemed a domain by this +// code, since according to the definition above, it would be +// evaluated as a public suffix. The caller is therefore responsible to +// decide how to further interpret such public suffix. +// +// `hostname` must be a valid ascii-based hostname. + +function getDomain(hostname) { + // A hostname starting with a dot is not a valid hostname. + if ( !hostname || hostname.charAt(0) === '.' ) { + return ''; + } + hostname = hostname.toLowerCase(); + var suffix = getPublicSuffix(hostname); + if ( suffix === hostname ) { + return ''; + } + var pos = hostname.lastIndexOf('.', hostname.lastIndexOf('.', hostname.length - suffix.length) - 1); + if ( pos <= 0 ) { + return hostname; + } + return hostname.slice(pos + 1); +} + +/******************************************************************************/ + +// Return longest public suffix. +// +// `hostname` must be a valid ascii-based string which respect hostname naming. + +function getPublicSuffix(hostname) { + if ( !hostname ) { + return ''; + } + // Since we slice down the hostname with each pass, the first match + // is the longest, so no need to find all the matching rules. + var pos; + while ( true ) { + pos = hostname.indexOf('.'); + if ( pos < 0 ) { + return hostname; + } + if ( search(exceptions, hostname) ) { + return hostname.slice(pos + 1); + } + if ( search(rules, hostname) ) { + return hostname; + } + if ( search(rules, '*' + hostname.slice(pos)) ) { + return hostname; + } + hostname = hostname.slice(pos + 1); + } + // unreachable +} + +/******************************************************************************/ + +// Look up a specific hostname. + +function search(store, hostname) { + // Extract TLD + var pos = hostname.lastIndexOf('.'); + var tld, remainder; + if ( pos < 0 ) { + tld = hostname; + remainder = hostname; + } else { + tld = hostname.slice(pos + 1); + remainder = hostname.slice(0, pos); + } + var substore = store.get(tld); + if ( substore === undefined ) { return false; } + // If substore is a string, use indexOf() + if ( typeof substore === 'string' ) { + return substore.indexOf(' ' + remainder + ' ') >= 0; + } + // It is an array: use binary search. + var l = remainder.length; + if ( l >= substore.length ) { return false; } + var haystack = substore[l]; + if ( haystack.length === 0 ) { return false; } + var left = 0; + var right = Math.floor(haystack.length / l + 0.5); + var i, needle; + while ( left < right ) { + i = left + right >> 1; + needle = haystack.substr( l * i, l ); + if ( remainder < needle ) { + right = i; + } else if ( remainder > needle ) { + left = i + 1; + } else { + return true; + } + } + return false; +} + +/******************************************************************************/ + +// Parse and set a UTF-8 text-based suffix list. Format is same as found at: +// http://publicsuffix.org/list/ +// +// `toAscii` is a converter from unicode to punycode. Required since the +// Public Suffix List contains unicode characters. +// Suggestion: use it's quite good. + +function parse(text, toAscii) { + exceptions = new Map(); + rules = new Map(); + + // http://publicsuffix.org/list/: + // "... all rules must be canonicalized in the normal way + // for hostnames - lower-case, Punycode ..." + text = text.toLowerCase(); + + var lineBeg = 0, lineEnd; + var textEnd = text.length; + var line, store, pos, tld; + + while ( lineBeg < textEnd ) { + lineEnd = text.indexOf('\n', lineBeg); + if ( lineEnd < 0 ) { + lineEnd = text.indexOf('\r', lineBeg); + if ( lineEnd < 0 ) { + lineEnd = textEnd; + } + } + line = text.slice(lineBeg, lineEnd).trim(); + lineBeg = lineEnd + 1; + + if ( line.length === 0 ) { + continue; + } + + // Ignore comments + pos = line.indexOf('//'); + if ( pos >= 0 ) { + line = line.slice(0, pos); + } + + // Ignore surrounding whitespaces + line = line.trim(); + if ( !line ) { + continue; + } + + if ( mustPunycode.test(line) ) { + line = toAscii(line); + } + + // Is this an exception rule? + if ( line.charAt(0) === '!' ) { + store = exceptions; + line = line.slice(1); + } else { + store = rules; + } + + // Extract TLD + pos = line.lastIndexOf('.'); + if ( pos < 0 ) { + tld = line; + } else { + tld = line.slice(pos + 1); + line = line.slice(0, pos); + } + + // Store suffix using tld as key + var substore = store.get(tld); + if ( substore === undefined ) { + store.set(tld, (substore = [])); + } + if ( line ) { + substore.push(line); + } + } + crystallize(exceptions); + crystallize(rules); + + window.dispatchEvent(new CustomEvent('publicSuffixList')); +} + +/******************************************************************************/ + +// Cristallize the storage of suffixes using optimal internal representation +// for future look up. + +function crystallize(store) { + for ( var entry of store ) { + var tld = entry[0]; + var suffixes = entry[1]; + // No suffix + if ( suffixes.length === 0 ) { + store.set(tld, ''); + continue; + } + // Concatenated list of suffixes less than cutoff length: + // Store as string, lookup using indexOf() + var s = suffixes.join(' '); + if ( s.length < cutoffLength ) { + store.set(tld, ' ' + s + ' '); + continue; + } + // Concatenated list of suffixes greater or equal to cutoff length + // Store as array keyed on suffix length, lookup using binary search. + // I borrowed the idea to key on string length here: + // http://ejohn.org/blog/dictionary-lookups-in-javascript/#comment-392072 + var i = suffixes.length, l; + var aa = []; + while ( i-- ) { + var suffix = suffixes[i]; + var j = aa.length; + l = suffix.length; + while ( j <= l ) { + aa[j] = []; j += 1; + } + aa[l].push(suffix); + } + l = aa.length; + while ( l-- ) { + aa[l] = aa[l].sort().join(''); + } + store.set(tld, aa); + } + return store; +} + +/******************************************************************************/ + +const selfieMagic = 1; + +function toSelfie() { + return { + magic: selfieMagic, + rules: Array.from(rules), + exceptions: Array.from(exceptions) + }; +} + +function fromSelfie(selfie) { + if ( typeof selfie !== 'object' || selfie.magic !== selfieMagic ) { + return false; + } + rules = new Map(selfie.rules); + exceptions = new Map(selfie.exceptions); + window.dispatchEvent(new CustomEvent('publicSuffixList')); + return true; +} + +/******************************************************************************/ + +// Public API + +root = root || window; + +root.publicSuffixList = { + version: '1.0', + parse: parse, + getDomain: getDomain, + getPublicSuffix: getPublicSuffix, + toSelfie: toSelfie, + fromSelfie: fromSelfie, + get empty() { + return rules.size === 0; + } +}; + +/******************************************************************************/ + +})(this); + diff --git a/restscrape/uBlock/lib/punycode.js b/restscrape/uBlock/lib/punycode.js new file mode 100644 index 0000000..f3866b8 --- /dev/null +++ b/restscrape/uBlock/lib/punycode.js @@ -0,0 +1,530 @@ +/*! https://mths.be/punycode v1.3.2 by @mathias */ +;(function(root) { + + /** Detect free variables */ + var freeExports = typeof exports == 'object' && exports && + !exports.nodeType && exports; + var freeModule = typeof module == 'object' && module && + !module.nodeType && module; + var freeGlobal = typeof global == 'object' && global; + if ( + freeGlobal.global === freeGlobal || + freeGlobal.window === freeGlobal || + freeGlobal.self === freeGlobal + ) { + root = freeGlobal; + } + + /** + * The `punycode` object. + * @name punycode + * @type Object + */ + var punycode, + + /** Highest positive signed 32-bit float value */ + maxInt = 2147483647, // aka. 0x7FFFFFFF or 2^31-1 + + /** Bootstring parameters */ + base = 36, + tMin = 1, + tMax = 26, + skew = 38, + damp = 700, + initialBias = 72, + initialN = 128, // 0x80 + delimiter = '-', // '\x2D' + + /** Regular expressions */ + regexPunycode = /^xn--/, + regexNonASCII = /[^\x20-\x7E]/, // unprintable ASCII chars + non-ASCII chars + regexSeparators = /[\x2E\u3002\uFF0E\uFF61]/g, // RFC 3490 separators + + /** Error messages */ + errors = { + 'overflow': 'Overflow: input needs wider integers to process', + 'not-basic': 'Illegal input >= 0x80 (not a basic code point)', + 'invalid-input': 'Invalid input' + }, + + /** Convenience shortcuts */ + baseMinusTMin = base - tMin, + floor = Math.floor, + stringFromCharCode = String.fromCharCode, + + /** Temporary variable */ + key; + + /*--------------------------------------------------------------------------*/ + + /** + * A generic error utility function. + * @private + * @param {String} type The error type. + * @returns {Error} Throws a `RangeError` with the applicable error message. + */ + function error(type) { + throw new RangeError(errors[type]); + } + + /** + * A generic `Array#map` utility function. + * @private + * @param {Array} array The array to iterate over. + * @param {Function} callback The function that gets called for every array + * item. + * @returns {Array} A new array of values returned by the callback function. + */ + function map(array, fn) { + var length = array.length; + var result = []; + while (length--) { + result[length] = fn(array[length]); + } + return result; + } + + /** + * A simple `Array#map`-like wrapper to work with domain name strings or email + * addresses. + * @private + * @param {String} domain The domain name or email address. + * @param {Function} callback The function that gets called for every + * character. + * @returns {Array} A new string of characters returned by the callback + * function. + */ + function mapDomain(string, fn) { + var parts = string.split('@'); + var result = ''; + if (parts.length > 1) { + // In email addresses, only the domain name should be punycoded. Leave + // the local part (i.e. everything up to `@`) intact. + result = parts[0] + '@'; + string = parts[1]; + } + // Avoid `split(regex)` for IE8 compatibility. See #17. + string = string.replace(regexSeparators, '\x2E'); + var labels = string.split('.'); + var encoded = map(labels, fn).join('.'); + return result + encoded; + } + + /** + * Creates an array containing the numeric code points of each Unicode + * character in the string. While JavaScript uses UCS-2 internally, + * this function will convert a pair of surrogate halves (each of which + * UCS-2 exposes as separate characters) into a single code point, + * matching UTF-16. + * @see `punycode.ucs2.encode` + * @see + * @memberOf punycode.ucs2 + * @name decode + * @param {String} string The Unicode input string (UCS-2). + * @returns {Array} The new array of code points. + */ + function ucs2decode(string) { + var output = [], + counter = 0, + length = string.length, + value, + extra; + while (counter < length) { + value = string.charCodeAt(counter++); + if (value >= 0xD800 && value <= 0xDBFF && counter < length) { + // high surrogate, and there is a next character + extra = string.charCodeAt(counter++); + if ((extra & 0xFC00) == 0xDC00) { // low surrogate + output.push(((value & 0x3FF) << 10) + (extra & 0x3FF) + 0x10000); + } else { + // unmatched surrogate; only append this code unit, in case the next + // code unit is the high surrogate of a surrogate pair + output.push(value); + counter--; + } + } else { + output.push(value); + } + } + return output; + } + + /** + * Creates a string based on an array of numeric code points. + * @see `punycode.ucs2.decode` + * @memberOf punycode.ucs2 + * @name encode + * @param {Array} codePoints The array of numeric code points. + * @returns {String} The new Unicode string (UCS-2). + */ + function ucs2encode(array) { + return map(array, function(value) { + var output = ''; + if (value > 0xFFFF) { + value -= 0x10000; + output += stringFromCharCode(value >>> 10 & 0x3FF | 0xD800); + value = 0xDC00 | value & 0x3FF; + } + output += stringFromCharCode(value); + return output; + }).join(''); + } + + /** + * Converts a basic code point into a digit/integer. + * @see `digitToBasic()` + * @private + * @param {Number} codePoint The basic numeric code point value. + * @returns {Number} The numeric value of a basic code point (for use in + * representing integers) in the range `0` to `base - 1`, or `base` if + * the code point does not represent a value. + */ + function basicToDigit(codePoint) { + if (codePoint - 48 < 10) { + return codePoint - 22; + } + if (codePoint - 65 < 26) { + return codePoint - 65; + } + if (codePoint - 97 < 26) { + return codePoint - 97; + } + return base; + } + + /** + * Converts a digit/integer into a basic code point. + * @see `basicToDigit()` + * @private + * @param {Number} digit The numeric value of a basic code point. + * @returns {Number} The basic code point whose value (when used for + * representing integers) is `digit`, which needs to be in the range + * `0` to `base - 1`. If `flag` is non-zero, the uppercase form is + * used; else, the lowercase form is used. The behavior is undefined + * if `flag` is non-zero and `digit` has no uppercase form. + */ + function digitToBasic(digit, flag) { + // 0..25 map to ASCII a..z or A..Z + // 26..35 map to ASCII 0..9 + return digit + 22 + 75 * (digit < 26) - ((flag != 0) << 5); + } + + /** + * Bias adaptation function as per section 3.4 of RFC 3492. + * http://tools.ietf.org/html/rfc3492#section-3.4 + * @private + */ + function adapt(delta, numPoints, firstTime) { + var k = 0; + delta = firstTime ? floor(delta / damp) : delta >> 1; + delta += floor(delta / numPoints); + for (/* no initialization */; delta > baseMinusTMin * tMax >> 1; k += base) { + delta = floor(delta / baseMinusTMin); + } + return floor(k + (baseMinusTMin + 1) * delta / (delta + skew)); + } + + /** + * Converts a Punycode string of ASCII-only symbols to a string of Unicode + * symbols. + * @memberOf punycode + * @param {String} input The Punycode string of ASCII-only symbols. + * @returns {String} The resulting string of Unicode symbols. + */ + function decode(input) { + // Don't use UCS-2 + var output = [], + inputLength = input.length, + out, + i = 0, + n = initialN, + bias = initialBias, + basic, + j, + index, + oldi, + w, + k, + digit, + t, + /** Cached calculation results */ + baseMinusT; + + // Handle the basic code points: let `basic` be the number of input code + // points before the last delimiter, or `0` if there is none, then copy + // the first basic code points to the output. + + basic = input.lastIndexOf(delimiter); + if (basic < 0) { + basic = 0; + } + + for (j = 0; j < basic; ++j) { + // if it's not a basic code point + if (input.charCodeAt(j) >= 0x80) { + error('not-basic'); + } + output.push(input.charCodeAt(j)); + } + + // Main decoding loop: start just after the last delimiter if any basic code + // points were copied; start at the beginning otherwise. + + for (index = basic > 0 ? basic + 1 : 0; index < inputLength; /* no final expression */) { + + // `index` is the index of the next character to be consumed. + // Decode a generalized variable-length integer into `delta`, + // which gets added to `i`. The overflow checking is easier + // if we increase `i` as we go, then subtract off its starting + // value at the end to obtain `delta`. + for (oldi = i, w = 1, k = base; /* no condition */; k += base) { + + if (index >= inputLength) { + error('invalid-input'); + } + + digit = basicToDigit(input.charCodeAt(index++)); + + if (digit >= base || digit > floor((maxInt - i) / w)) { + error('overflow'); + } + + i += digit * w; + t = k <= bias ? tMin : (k >= bias + tMax ? tMax : k - bias); + + if (digit < t) { + break; + } + + baseMinusT = base - t; + if (w > floor(maxInt / baseMinusT)) { + error('overflow'); + } + + w *= baseMinusT; + + } + + out = output.length + 1; + bias = adapt(i - oldi, out, oldi == 0); + + // `i` was supposed to wrap around from `out` to `0`, + // incrementing `n` each time, so we'll fix that now: + if (floor(i / out) > maxInt - n) { + error('overflow'); + } + + n += floor(i / out); + i %= out; + + // Insert `n` at position `i` of the output + output.splice(i++, 0, n); + + } + + return ucs2encode(output); + } + + /** + * Converts a string of Unicode symbols (e.g. a domain name label) to a + * Punycode string of ASCII-only symbols. + * @memberOf punycode + * @param {String} input The string of Unicode symbols. + * @returns {String} The resulting Punycode string of ASCII-only symbols. + */ + function encode(input) { + var n, + delta, + handledCPCount, + basicLength, + bias, + j, + m, + q, + k, + t, + currentValue, + output = [], + /** `inputLength` will hold the number of code points in `input`. */ + inputLength, + /** Cached calculation results */ + handledCPCountPlusOne, + baseMinusT, + qMinusT; + + // Convert the input in UCS-2 to Unicode + input = ucs2decode(input); + + // Cache the length + inputLength = input.length; + + // Initialize the state + n = initialN; + delta = 0; + bias = initialBias; + + // Handle the basic code points + for (j = 0; j < inputLength; ++j) { + currentValue = input[j]; + if (currentValue < 0x80) { + output.push(stringFromCharCode(currentValue)); + } + } + + handledCPCount = basicLength = output.length; + + // `handledCPCount` is the number of code points that have been handled; + // `basicLength` is the number of basic code points. + + // Finish the basic string - if it is not empty - with a delimiter + if (basicLength) { + output.push(delimiter); + } + + // Main encoding loop: + while (handledCPCount < inputLength) { + + // All non-basic code points < n have been handled already. Find the next + // larger one: + for (m = maxInt, j = 0; j < inputLength; ++j) { + currentValue = input[j]; + if (currentValue >= n && currentValue < m) { + m = currentValue; + } + } + + // Increase `delta` enough to advance the decoder's state to , + // but guard against overflow + handledCPCountPlusOne = handledCPCount + 1; + if (m - n > floor((maxInt - delta) / handledCPCountPlusOne)) { + error('overflow'); + } + + delta += (m - n) * handledCPCountPlusOne; + n = m; + + for (j = 0; j < inputLength; ++j) { + currentValue = input[j]; + + if (currentValue < n && ++delta > maxInt) { + error('overflow'); + } + + if (currentValue == n) { + // Represent delta as a generalized variable-length integer + for (q = delta, k = base; /* no condition */; k += base) { + t = k <= bias ? tMin : (k >= bias + tMax ? tMax : k - bias); + if (q < t) { + break; + } + qMinusT = q - t; + baseMinusT = base - t; + output.push( + stringFromCharCode(digitToBasic(t + qMinusT % baseMinusT, 0)) + ); + q = floor(qMinusT / baseMinusT); + } + + output.push(stringFromCharCode(digitToBasic(q, 0))); + bias = adapt(delta, handledCPCountPlusOne, handledCPCount == basicLength); + delta = 0; + ++handledCPCount; + } + } + + ++delta; + ++n; + + } + return output.join(''); + } + + /** + * Converts a Punycode string representing a domain name or an email address + * to Unicode. Only the Punycoded parts of the input will be converted, i.e. + * it doesn't matter if you call it on a string that has already been + * converted to Unicode. + * @memberOf punycode + * @param {String} input The Punycoded domain name or email address to + * convert to Unicode. + * @returns {String} The Unicode representation of the given Punycode + * string. + */ + function toUnicode(input) { + return mapDomain(input, function(string) { + return regexPunycode.test(string) + ? decode(string.slice(4).toLowerCase()) + : string; + }); + } + + /** + * Converts a Unicode string representing a domain name or an email address to + * Punycode. Only the non-ASCII parts of the domain name will be converted, + * i.e. it doesn't matter if you call it with a domain that's already in + * ASCII. + * @memberOf punycode + * @param {String} input The domain name or email address to convert, as a + * Unicode string. + * @returns {String} The Punycode representation of the given domain name or + * email address. + */ + function toASCII(input) { + return mapDomain(input, function(string) { + return regexNonASCII.test(string) + ? 'xn--' + encode(string) + : string; + }); + } + + /*--------------------------------------------------------------------------*/ + + /** Define the public API */ + punycode = { + /** + * A string representing the current Punycode.js version number. + * @memberOf punycode + * @type String + */ + 'version': '1.3.2', + /** + * An object of methods to convert from JavaScript's internal character + * representation (UCS-2) to Unicode code points, and back. + * @see + * @memberOf punycode + * @type Object + */ + 'ucs2': { + 'decode': ucs2decode, + 'encode': ucs2encode + }, + 'decode': decode, + 'encode': encode, + 'toASCII': toASCII, + 'toUnicode': toUnicode + }; + + /** Expose `punycode` */ + // Some AMD build optimizers, like r.js, check for specific condition patterns + // like the following: + if ( + typeof define == 'function' && + typeof define.amd == 'object' && + define.amd + ) { + define('punycode', function() { + return punycode; + }); + } else if (freeExports && freeModule) { + if (module.exports == freeExports) { // in Node.js or RingoJS v0.8.0+ + freeModule.exports = punycode; + } else { // in Narwhal or RingoJS v0.7.0- + for (key in punycode) { + punycode.hasOwnProperty(key) && (freeExports[key] = punycode[key]); + } + } + } else { // in Rhino or a web browser + root.punycode = punycode; + } + +}(this)); diff --git a/restscrape/uBlock/logger-ui.html b/restscrape/uBlock/logger-ui.html new file mode 100644 index 0000000..fda4df5 --- /dev/null +++ b/restscrape/uBlock/logger-ui.html @@ -0,0 +1,204 @@ + + + + + + + + + + + + + + + +
+
+ + refresh + code + +
+ +
+ +
+
+
+
+ double-angle-up + double-angle-left + + eraser + floppy-o +
+
+
+
    +
    +
    +
    +
    +
    + double-angle-up + times + eraser + pause-circle-oplay-circle-o + + filter + + + angle-up +
    +
    +
    css/fontimagescriptxhrframedom
    +
    +
    +     +     +
    +
    + clipboard + cog +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    00:00:00 ** 3,3inline-script 
    +
    +
    +
    +
    + +
    + +
    +
    +
    +
    +
    +
    + + + + + + + + + + + + + + diff --git a/restscrape/uBlock/popup.html b/restscrape/uBlock/popup.html new file mode 100644 index 0000000..57366f7 --- /dev/null +++ b/restscrape/uBlock/popup.html @@ -0,0 +1,75 @@ + + + + + + + + + + +uBlock Origin + + + +
       
    +
    +
    +

    power-off

    +

    + bolt + eye-dropper + list-alt + sliders +

    +

     

    +

    +   +

    +

    ?

    +
    refresh
    +

    +   +

    +

    ?

    +

     

    +

     

    +
    + files-o + film + eye-slash + font + code +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    lockeraser
    +
    +
    + + + + + + + + + + + + + + + diff --git a/restscrape/uBlock/settings.html b/restscrape/uBlock/settings.html new file mode 100644 index 0000000..8403731 --- /dev/null +++ b/restscrape/uBlock/settings.html @@ -0,0 +1,71 @@ + + + + + +uBlock — Settings + + + + + + +
    + + +
    +
    +
      +
    • +
      • +
      • +
      +
      • +
      • +
      • +
      +
    +
    + +
    +

      + +

    +

    +
    + + + + + + + + + + + + + diff --git a/restscrape/uBlock/shortcuts.html b/restscrape/uBlock/shortcuts.html new file mode 100644 index 0000000..22a7238 --- /dev/null +++ b/restscrape/uBlock/shortcuts.html @@ -0,0 +1,37 @@ + + + + + +uBlock Origin — Keyboard shortcuts + + + + + + + +
    +
    +
    + +