From 152ea51d832decef6370653d42a36345dea4da68 Mon Sep 17 00:00:00 2001 From: biomedtech Date: Mon, 20 Sep 2021 00:53:26 +0200 Subject: [PATCH 001/137] feat: select module to run unit tests on --- test/testunit.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/testunit.js b/test/testunit.js index 6fbeb5664b..c582b5d94f 100644 --- a/test/testunit.js +++ b/test/testunit.js @@ -3,7 +3,7 @@ const path = require('path'); const program = require('commander'); const reqskulpt = require('../support/run/require-skulpt').requireSkulpt; -function test (python3, opt) { +function test (python3, opt, module = undefined) { var startime, endtime, elapsed; // Import Skulpt @@ -45,6 +45,8 @@ function test (python3, opt) { let basename = path.basename(file, ".py"); if (stat.isFile() && basename.startsWith("test_") && (path.extname(file) == ".py")) { + if (module && !basename.endsWith(module)) continue; + modules.push([file, basename]); } } @@ -107,7 +109,8 @@ function test (python3, opt) { program .option('--python3', 'Python 3') .option('-o, --opt', 'use optimized skulpt') + .option('--module ', 'test specific module') .parse(process.argv); -test(program.python3, program.opt); +test(program.python3, program.opt, program.module); From 8778cf565c7d7fd38c6489081f0674a658a272eb Mon Sep 17 00:00:00 2001 From: stu Date: Thu, 3 Mar 2022 16:03:13 +0800 Subject: [PATCH 002/137] Fix small typo in error message --- src/function.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/function.js b/src/function.js index 82ef2332b3..55e4cbf817 100644 --- a/src/function.js +++ b/src/function.js @@ -231,7 +231,9 @@ function $resolveArgs(posargs, kw) { let vararg = (posargs.length > args.length) ? posargs.slice(args.length) : []; args[totalArgs] = new Sk.builtin.tuple(vararg); } else if (nposargs > co_argcount) { - throw new Sk.builtin.TypeError(this.$name + "() takes " + co_argcount + " positional argument" + (co_argcount == 1 ? "" : "s") + " but " + nposargs + (nposargs == 1 ? " was " : " were ") + " given"); + const plural_expected = co_argcount == 1 ? "argument" : "arguments"; + const plural_given = nposargs == 1 ? "was" : "were"; + throw new Sk.builtin.TypeError(`${this.$name}"() takes ${co_argcount} positional ${plural_expected} but ${nposargs} ${plural_given} given`); } /* Handle keyword arguments */ From 5f4eba136499f3d6147a19120b51304d7067586d Mon Sep 17 00:00:00 2001 From: stu Date: Wed, 9 Mar 2022 20:51:05 +0800 Subject: [PATCH 003/137] Fix: __eq__ set with no __hash__ makes a class unhashable --- src/type.js | 4 +++ test/unit3/test_class.py | 58 ++++++++++++++++++++-------------------- 2 files changed, 33 insertions(+), 29 deletions(-) diff --git a/src/type.js b/src/type.js index 197d87d753..894eb0bdb6 100644 --- a/src/type.js +++ b/src/type.js @@ -507,6 +507,10 @@ function $allocateSlots() { } }); } + if (proto.hasOwnProperty("__eq__") && !proto.hasOwnProperty("__hash__")) { + // https://docs.python.org/3/reference/datamodel.html#object.__hash__ + proto.tp$hash = proto.__hash__ = Sk.builtin.none.none$; + } } function $allocateSlot(dunder, dunderFunc) { diff --git a/test/unit3/test_class.py b/test/unit3/test_class.py index a793201892..2d3a43673c 100644 --- a/test/unit3/test_class.py +++ b/test/unit3/test_class.py @@ -385,35 +385,35 @@ def index(x): self.assertRaises(TypeError, index, BadTypeClass()) - # def testHashStuff(self): - # # Test correct errors from hash() on objects with comparisons but - # # no __hash__ - # - # class C0: - # pass - # - # hash(C0()) # This should work; the next two should raise TypeError - # - # class C2: - # def __eq__(self, other): return 1 - # - # self.assertRaises(TypeError, hash, C2()) - - - # def testSFBug532646(self): - # # Test for SF bug 532646 - # - # class A: - # pass - # A.__call__ = A() - # a = A() - # - # try: - # a() # This should not segfault - # except RecursionError: - # pass - # else: - # self.fail("Failed to raise RecursionError") + def testHashStuff(self): + # Test correct errors from hash() on objects with comparisons but + # no __hash__ + + class C0: + pass + + hash(C0()) # This should work; the next two should raise TypeError + + class C2: + def __eq__(self, other): return 1 + + self.assertRaises(TypeError, hash, C2()) + + + def testSFBug532646(self): + # Test for SF bug 532646 + + class A: + pass + A.__call__ = A() + a = A() + + try: + a() # This should not segfault + except RecursionError: + pass + else: + self.fail("Failed to raise RecursionError") # def testForExceptionsRaisedInInstanceGetattr2(self): # # Tests for exceptions raised in instance_getattr2(). From c30413017c1035ffaaf5d716ec90447d6a31a131 Mon Sep 17 00:00:00 2001 From: stu Date: Thu, 24 Mar 2022 18:07:07 +0800 Subject: [PATCH 004/137] make circular import errors work with current import implementation --- src/compile.js | 2 +- src/import.js | 72 +++++++++++---- src/module.js | 11 +++ test/testunit.js | 17 +++- test/unit3/test_import/__init__.py | 91 +++++++++++++++++++ test/unit3/test_import/data/__init__.py | 0 .../data/circular_imports/__init__.py | 0 .../data/circular_imports/basic.py | 2 + .../data/circular_imports/basic2.py | 1 + .../data/circular_imports/binding.py | 1 + .../data/circular_imports/binding2.py | 1 + .../data/circular_imports/from_cycle1.py | 2 + .../data/circular_imports/from_cycle2.py | 2 + .../data/circular_imports/indirect.py | 1 + .../data/circular_imports/rebinding.py | 3 + .../data/circular_imports/rebinding2.py | 3 + .../data/circular_imports/source.py | 2 + .../data/circular_imports/subpackage.py | 2 + .../data/circular_imports/subpkg/__init__.py | 0 .../circular_imports/subpkg/subpackage2.py | 2 + .../data/circular_imports/subpkg/util.py | 2 + .../test_import/data/circular_imports/use.py | 2 + .../test_import/data/circular_imports/util.py | 2 + .../test_import/data/package/__init__.py | 2 + .../test_import/data/package/submodule.py | 0 .../test_import/data/package2/__init__.py | 0 .../test_import/data/package2/submodule1.py | 3 + .../test_import/data/package2/submodule2.py | 0 .../test_import/data/unwritable/__init__.py | 12 +++ test/unit3/test_import/data/unwritable/x.py | 0 30 files changed, 215 insertions(+), 23 deletions(-) create mode 100644 test/unit3/test_import/__init__.py create mode 100644 test/unit3/test_import/data/__init__.py create mode 100644 test/unit3/test_import/data/circular_imports/__init__.py create mode 100644 test/unit3/test_import/data/circular_imports/basic.py create mode 100644 test/unit3/test_import/data/circular_imports/basic2.py create mode 100644 test/unit3/test_import/data/circular_imports/binding.py create mode 100644 test/unit3/test_import/data/circular_imports/binding2.py create mode 100644 test/unit3/test_import/data/circular_imports/from_cycle1.py create mode 100644 test/unit3/test_import/data/circular_imports/from_cycle2.py create mode 100644 test/unit3/test_import/data/circular_imports/indirect.py create mode 100644 test/unit3/test_import/data/circular_imports/rebinding.py create mode 100644 test/unit3/test_import/data/circular_imports/rebinding2.py create mode 100644 test/unit3/test_import/data/circular_imports/source.py create mode 100644 test/unit3/test_import/data/circular_imports/subpackage.py create mode 100644 test/unit3/test_import/data/circular_imports/subpkg/__init__.py create mode 100644 test/unit3/test_import/data/circular_imports/subpkg/subpackage2.py create mode 100644 test/unit3/test_import/data/circular_imports/subpkg/util.py create mode 100644 test/unit3/test_import/data/circular_imports/use.py create mode 100644 test/unit3/test_import/data/circular_imports/util.py create mode 100644 test/unit3/test_import/data/package/__init__.py create mode 100644 test/unit3/test_import/data/package/submodule.py create mode 100644 test/unit3/test_import/data/package2/__init__.py create mode 100644 test/unit3/test_import/data/package2/submodule1.py create mode 100644 test/unit3/test_import/data/package2/submodule2.py create mode 100644 test/unit3/test_import/data/unwritable/__init__.py create mode 100644 test/unit3/test_import/data/unwritable/x.py diff --git a/src/compile.js b/src/compile.js index d8dd72e69b..e6185642bc 100644 --- a/src/compile.js +++ b/src/compile.js @@ -1837,7 +1837,7 @@ Compiler.prototype.cfromimport = function (s) { level = -1; } for (i = 0; i < n; ++i) { - names[i] = "'" + fixReserved(s.names[i].name.v) + "'"; + names[i] = "'" + s.names[i].name.v + "'"; } out("$ret = Sk.builtin.__import__(", s.module["$r"]().v, ",$gbl,$loc,[", names, "],",level,");"); diff --git a/src/import.js b/src/import.js index f9cf1ad6d5..4551a1a8d6 100644 --- a/src/import.js +++ b/src/import.js @@ -212,8 +212,6 @@ Sk.importModuleInternal_ = function (name, dumpJS, modname, suppliedPyBody, rela return undefined; } - // Now we know this module exists, we can add it to the cache - Sk.sysmodules.mp$ass_subscript(new Sk.builtin.str(modname), module); module.$js = co.code; // todo; only in DEBUG? finalcode = co.code; @@ -258,29 +256,55 @@ Sk.importModuleInternal_ = function (name, dumpJS, modname, suppliedPyBody, rela finalcode += "\n" + co.funcname + ";"; } + const pyModName = new Sk.builtin.str(modname); + const pyAttrName = new Sk.builtin.str(name); + + // Now we know this module exists, we can add it to the cache + Sk.sysmodules.mp$ass_subscript(pyModName, module); + if (relativeToPackage) { + relativeToPackage.tp$setattr(pyAttrName, module); + } + modscope = Sk.global["eval"](finalcode); + module.init$dict(pyModName, Sk.builtin.none.none$); + + module["$d"].__package__ = co.packagePath + ? pyModName + : parentModName + ? new Sk.builtin.str(absolutePackagePrefix + parentModName) + : relativePackageName + ? relativePackageName + : Sk.builtin.none.none$; - module["$d"] = { - "__name__": new Sk.builtin.str(modname), - "__doc__": Sk.builtin.none.none$, - "__package__": co.packagePath ? new Sk.builtin.str(modname) : - parentModName ? new Sk.builtin.str(absolutePackagePrefix + parentModName) : - relativePackageName ? relativePackageName : Sk.builtin.none.none$ - }; if (co.packagePath) { module["$d"]["__path__"] = new Sk.builtin.tuple([new Sk.builtin.str(co.packagePath)]); } if (co.filename && co.funcname !== "$builtinmodule") { module["$d"]["__file__"] = new Sk.builtin.str(co.filename); } + const packageInitializing = relativeToPackage && relativeToPackage.$initializing; + module.$initializing = true; + if (relativeToPackage && !packageInitializing) { + relativeToPackage.$initializing = true; + } try { return modscope(module["$d"]); } catch (e) { try { // don't cache a module if it raised an exception on load - Sk.abstr.objectDelItem(Sk.sysmodules, new Sk.builtin.str(modname)); + Sk.abstr.objectDelItem(Sk.sysmodules, pyModName); } catch {} + if (relativeToPackage) { + try { + relativeToPackage.tp$setattr(pyAttrName, undefined); + } catch {} + } throw e; + } finally { + module.$initializing = false; + if (relativeToPackage && !packageInitializing) { + relativeToPackage.$initializing = false; + } } }, function (modlocs) { var i; @@ -323,9 +347,6 @@ Sk.importModuleInternal_ = function (name, dumpJS, modname, suppliedPyBody, rela return topLevelModuleToReturn; } - if (relativeToPackage) { - relativeToPackage.tp$setattr(new Sk.builtin.str(name), module); - } //print("name", name, "modname", modname, "returning leaf"); // otherwise we return the actual module that we just imported @@ -468,7 +489,7 @@ Sk.builtin.__import__ = function (name, globals, locals, fromlist, level) { var i; var fromName; var leafModule; - var importChain; + const chainedImports = [null]; leafModule = Sk.sysmodules.mp$subscript( new Sk.builtin.str((relativeToPackageName || "") + @@ -480,14 +501,29 @@ Sk.builtin.__import__ = function (name, globals, locals, fromlist, level) { // "ret" is the module we're importing from // Only import from file system if we have not found the fromName in the current module - if (fromName != "*" && leafModule.tp$getattr(new Sk.builtin.str(fromName)) === undefined) { - importChain = Sk.misceval.chain(importChain, - Sk.importModuleInternal_.bind(null, fromName, undefined, undefined, undefined, leafModule, true, true) + if (fromName != "*") { + // use hasattr to support lazy module attributes + // and modules.tp$getattr might throw if still initializing + chainedImports.push( + () => Sk.builtin.hasattr(leafModule, new Sk.builtin.str(fromName)), + (rv) => { + if (!Sk.misceval.isTrue(rv)) { + return Sk.importModuleInternal_( + fromName, + undefined, + undefined, + undefined, + leafModule, + true, + true + ); + } + } ); } } - return Sk.misceval.chain(importChain, function() { + return Sk.misceval.chain(...chainedImports, function () { // if there's a fromlist we want to return the leaf module // (ret), not the toplevel namespace Sk.asserts.assert(leafModule); diff --git a/src/module.js b/src/module.js index 6b7c62f3fd..8c85b3b9b8 100644 --- a/src/module.js +++ b/src/module.js @@ -37,6 +37,8 @@ Sk.builtin.module = Sk.abstr.buildNativeClass("module", { } ); return canSuspend ? res : Sk.misceval.retryOptionalSuspensionOrThrow(res); + } else { + this.check$initializing(pyName); } }, tp$setattr: Sk.generic.setAttr, @@ -89,6 +91,15 @@ Sk.builtin.module = Sk.abstr.buildNativeClass("module", { }, }, proto: { + check$initializing(attr) { + if (this.$initializing) { + const msg = + "partially initialized " + + `module ${this.get$name()} has no attribute '${attr.toString()}' ` + + "(most likely due to a circular import)"; + throw new Sk.builtin.AttributeError(msg); + } + }, init$dict(name, doc) { this.$d.__name__ = name; this.$d.__doc__ = doc; diff --git a/test/testunit.js b/test/testunit.js index c582b5d94f..098ad05106 100644 --- a/test/testunit.js +++ b/test/testunit.js @@ -40,14 +40,24 @@ function test (python3, opt, module = undefined) { var modules = []; for (var idx = 0; idx < files.length; idx++) { - let file = dir + '/' + files[idx]; + let file = dir + "/" + files[idx]; let stat = fs.statSync(file); let basename = path.basename(file, ".py"); - if (stat.isFile() && basename.startsWith("test_") && (path.extname(file) == ".py")) { - if (module && !basename.endsWith(module)) continue; + if (stat.isFile() && basename.startsWith("test_") && path.extname(file) == ".py") { + if (module && !basename.endsWith(module)) { + continue; + } modules.push([file, basename]); + } else if (stat.isDirectory() && basename.startsWith("test_")) { + if (!fs.statSync(file + "/__init__.py").isFile()) { + continue; + } + if (module && !basename.endsWith(module)) { + continue; + } + modules.push([file + ".py", path.basename(file + ".py", ".py")]); } } @@ -113,4 +123,3 @@ program .parse(process.argv); test(program.python3, program.opt, program.module); - diff --git a/test/unit3/test_import/__init__.py b/test/unit3/test_import/__init__.py new file mode 100644 index 0000000000..7b6b6fe770 --- /dev/null +++ b/test/unit3/test_import/__init__.py @@ -0,0 +1,91 @@ +import sys +import unittest + + +class CircularImportTests(unittest.TestCase): + + """See the docstrings of the modules being imported for the purpose of the + test.""" + + def tearDown(self): + """Make sure no modules pre-exist in sys.modules which are being used to + test.""" + for key in list(sys.modules.keys()): + if '.data' in key: + del sys.modules[key] + + def test_direct(self): + try: + from .data.circular_imports import basic + except ImportError: + self.fail('circular import through relative imports failed') + + def test_indirect(self): + try: + from .data.circular_imports import indirect + except ImportError: + self.fail('relative import in module contributing to circular ' + 'import failed') + + def test_subpackage(self): + try: + from .data.circular_imports import subpackage + except ImportError: + self.fail('circular import involving a subpackage failed') + + def test_rebinding(self): + try: + from .data.circular_imports import rebinding as rebinding + except ImportError: + self.fail('circular import with rebinding of module attribute failed') + from .data.circular_imports.subpkg import util + self.assertIs(util.util, rebinding.util) + + def test_binding(self): + try: + from .data.circular_imports import binding + except ImportError: + self.fail('circular import with binding a submodule to a name failed') + + def test_crossreference1(self): + from .data.circular_imports import use + from .data.circular_imports import source + + def test_crossreference2(self): + with self.assertRaises(AttributeError) as cm: + from .data.circular_imports import source + errmsg = str(cm.exception) + self.assertIn('.data.circular_imports.source', errmsg) + self.assertIn('spam', errmsg) + self.assertIn('partially initialized module', errmsg) + self.assertIn('circular import', errmsg) + + def test_circular_from_import(self): + # in python this is an import error + # with self.assertRaises(ImportError) as cm: + with self.assertRaises(AttributeError) as cm: + from .data.circular_imports import from_cycle1 + self.assertIn( + "partially initialized module ", + str(cm.exception) + ) + print(cm.exception) + self.assertIn(".circular_imports.from_cycle1'", str(cm.exception)) + self.assertIn("(most likely due to a circular import)", str(cm.exception)) + + # def test_unwritable_module(self): + # self.addCleanup(unload, "test.test_import.data.unwritable") + # self.addCleanup(unload, "test.test_import.data.unwritable.x") + + # from .data.unwritable import s unwritable + # with self.assertWarns(ImportWarning): + # from .data.unwritable import x + + # self.assertNotEqual(type(unwritable), ModuleType) + # self.assertEqual(type(x), ModuleType) + # with self.assertRaises(AttributeError): + # unwritable.x = 42 + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit3/test_import/data/__init__.py b/test/unit3/test_import/data/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/unit3/test_import/data/circular_imports/__init__.py b/test/unit3/test_import/data/circular_imports/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/unit3/test_import/data/circular_imports/basic.py b/test/unit3/test_import/data/circular_imports/basic.py new file mode 100644 index 0000000000..3e41e395db --- /dev/null +++ b/test/unit3/test_import/data/circular_imports/basic.py @@ -0,0 +1,2 @@ +"""Circular imports through direct, relative imports.""" +from . import basic2 diff --git a/test/unit3/test_import/data/circular_imports/basic2.py b/test/unit3/test_import/data/circular_imports/basic2.py new file mode 100644 index 0000000000..00bd2f29f3 --- /dev/null +++ b/test/unit3/test_import/data/circular_imports/basic2.py @@ -0,0 +1 @@ +from . import basic diff --git a/test/unit3/test_import/data/circular_imports/binding.py b/test/unit3/test_import/data/circular_imports/binding.py new file mode 100644 index 0000000000..a3a68eb3ac --- /dev/null +++ b/test/unit3/test_import/data/circular_imports/binding.py @@ -0,0 +1 @@ +from . import binding2 as binding2 \ No newline at end of file diff --git a/test/unit3/test_import/data/circular_imports/binding2.py b/test/unit3/test_import/data/circular_imports/binding2.py new file mode 100644 index 0000000000..f24597a24c --- /dev/null +++ b/test/unit3/test_import/data/circular_imports/binding2.py @@ -0,0 +1 @@ +from . import binding as binding diff --git a/test/unit3/test_import/data/circular_imports/from_cycle1.py b/test/unit3/test_import/data/circular_imports/from_cycle1.py new file mode 100644 index 0000000000..aacfd5f46f --- /dev/null +++ b/test/unit3/test_import/data/circular_imports/from_cycle1.py @@ -0,0 +1,2 @@ +from .from_cycle2 import a +b = 1 diff --git a/test/unit3/test_import/data/circular_imports/from_cycle2.py b/test/unit3/test_import/data/circular_imports/from_cycle2.py new file mode 100644 index 0000000000..62a66e1cfd --- /dev/null +++ b/test/unit3/test_import/data/circular_imports/from_cycle2.py @@ -0,0 +1,2 @@ +from .from_cycle1 import b +a = 1 diff --git a/test/unit3/test_import/data/circular_imports/indirect.py b/test/unit3/test_import/data/circular_imports/indirect.py new file mode 100644 index 0000000000..6925788d60 --- /dev/null +++ b/test/unit3/test_import/data/circular_imports/indirect.py @@ -0,0 +1 @@ +from . import basic, basic2 diff --git a/test/unit3/test_import/data/circular_imports/rebinding.py b/test/unit3/test_import/data/circular_imports/rebinding.py new file mode 100644 index 0000000000..2b77375559 --- /dev/null +++ b/test/unit3/test_import/data/circular_imports/rebinding.py @@ -0,0 +1,3 @@ +"""Test the binding of names when a circular import shares the same name as an +attribute.""" +from .rebinding2 import util diff --git a/test/unit3/test_import/data/circular_imports/rebinding2.py b/test/unit3/test_import/data/circular_imports/rebinding2.py new file mode 100644 index 0000000000..57a9e6945d --- /dev/null +++ b/test/unit3/test_import/data/circular_imports/rebinding2.py @@ -0,0 +1,3 @@ +from .subpkg import util +from . import rebinding +util = util.util diff --git a/test/unit3/test_import/data/circular_imports/source.py b/test/unit3/test_import/data/circular_imports/source.py new file mode 100644 index 0000000000..f104099048 --- /dev/null +++ b/test/unit3/test_import/data/circular_imports/source.py @@ -0,0 +1,2 @@ +from . import use +spam = 1 diff --git a/test/unit3/test_import/data/circular_imports/subpackage.py b/test/unit3/test_import/data/circular_imports/subpackage.py new file mode 100644 index 0000000000..7b412f76fc --- /dev/null +++ b/test/unit3/test_import/data/circular_imports/subpackage.py @@ -0,0 +1,2 @@ +"""Circular import involving a sub-package.""" +from .subpkg import subpackage2 diff --git a/test/unit3/test_import/data/circular_imports/subpkg/__init__.py b/test/unit3/test_import/data/circular_imports/subpkg/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/unit3/test_import/data/circular_imports/subpkg/subpackage2.py b/test/unit3/test_import/data/circular_imports/subpkg/subpackage2.py new file mode 100644 index 0000000000..17b893a1a2 --- /dev/null +++ b/test/unit3/test_import/data/circular_imports/subpkg/subpackage2.py @@ -0,0 +1,2 @@ +#from .util import util +from .. import subpackage diff --git a/test/unit3/test_import/data/circular_imports/subpkg/util.py b/test/unit3/test_import/data/circular_imports/subpkg/util.py new file mode 100644 index 0000000000..343bd843b2 --- /dev/null +++ b/test/unit3/test_import/data/circular_imports/subpkg/util.py @@ -0,0 +1,2 @@ +def util(): + pass diff --git a/test/unit3/test_import/data/circular_imports/use.py b/test/unit3/test_import/data/circular_imports/use.py new file mode 100644 index 0000000000..418f9e2688 --- /dev/null +++ b/test/unit3/test_import/data/circular_imports/use.py @@ -0,0 +1,2 @@ +from . import source +source.spam diff --git a/test/unit3/test_import/data/circular_imports/util.py b/test/unit3/test_import/data/circular_imports/util.py new file mode 100644 index 0000000000..343bd843b2 --- /dev/null +++ b/test/unit3/test_import/data/circular_imports/util.py @@ -0,0 +1,2 @@ +def util(): + pass diff --git a/test/unit3/test_import/data/package/__init__.py b/test/unit3/test_import/data/package/__init__.py new file mode 100644 index 0000000000..a4f2bc3400 --- /dev/null +++ b/test/unit3/test_import/data/package/__init__.py @@ -0,0 +1,2 @@ +import package.submodule +package.submodule diff --git a/test/unit3/test_import/data/package/submodule.py b/test/unit3/test_import/data/package/submodule.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/unit3/test_import/data/package2/__init__.py b/test/unit3/test_import/data/package2/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/unit3/test_import/data/package2/submodule1.py b/test/unit3/test_import/data/package2/submodule1.py new file mode 100644 index 0000000000..0698ed6de2 --- /dev/null +++ b/test/unit3/test_import/data/package2/submodule1.py @@ -0,0 +1,3 @@ +import sys +sys.modules.pop(__package__, None) +from . import submodule2 diff --git a/test/unit3/test_import/data/package2/submodule2.py b/test/unit3/test_import/data/package2/submodule2.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/unit3/test_import/data/unwritable/__init__.py b/test/unit3/test_import/data/unwritable/__init__.py new file mode 100644 index 0000000000..da4ddb3d02 --- /dev/null +++ b/test/unit3/test_import/data/unwritable/__init__.py @@ -0,0 +1,12 @@ +import sys + +class MyMod(object): + __slots__ = ['__builtins__', '__cached__', '__doc__', + '__file__', '__loader__', '__name__', + '__package__', '__path__', '__spec__'] + def __init__(self): + for attr in self.__slots__: + setattr(self, attr, globals()[attr]) + + +sys.modules[__name__] = MyMod() diff --git a/test/unit3/test_import/data/unwritable/x.py b/test/unit3/test_import/data/unwritable/x.py new file mode 100644 index 0000000000..e69de29bb2 From 3e6c78bf6cb79fe036ad47edede5c98bb2cb77cc Mon Sep 17 00:00:00 2001 From: stu Date: Thu, 24 Mar 2022 18:07:30 +0800 Subject: [PATCH 005/137] Make the module.tp$getattr still return undefined instead of raising --- src/import.js | 21 +++------------------ src/module.js | 19 ++++++------------- 2 files changed, 9 insertions(+), 31 deletions(-) diff --git a/src/import.js b/src/import.js index 4551a1a8d6..43992717e0 100644 --- a/src/import.js +++ b/src/import.js @@ -501,24 +501,9 @@ Sk.builtin.__import__ = function (name, globals, locals, fromlist, level) { // "ret" is the module we're importing from // Only import from file system if we have not found the fromName in the current module - if (fromName != "*") { - // use hasattr to support lazy module attributes - // and modules.tp$getattr might throw if still initializing - chainedImports.push( - () => Sk.builtin.hasattr(leafModule, new Sk.builtin.str(fromName)), - (rv) => { - if (!Sk.misceval.isTrue(rv)) { - return Sk.importModuleInternal_( - fromName, - undefined, - undefined, - undefined, - leafModule, - true, - true - ); - } - } + if (fromName != "*" && leafModule.tp$getattr(new Sk.builtin.str(fromName)) === undefined) { + chainedImports.push(() => + Sk.importModuleInternal_(fromName, undefined, undefined, undefined, leafModule, true, true) ); } } diff --git a/src/module.js b/src/module.js index 8c85b3b9b8..e740b4d839 100644 --- a/src/module.js +++ b/src/module.js @@ -37,8 +37,6 @@ Sk.builtin.module = Sk.abstr.buildNativeClass("module", { } ); return canSuspend ? res : Sk.misceval.retryOptionalSuspensionOrThrow(res); - } else { - this.check$initializing(pyName); } }, tp$setattr: Sk.generic.setAttr, @@ -91,15 +89,6 @@ Sk.builtin.module = Sk.abstr.buildNativeClass("module", { }, }, proto: { - check$initializing(attr) { - if (this.$initializing) { - const msg = - "partially initialized " + - `module ${this.get$name()} has no attribute '${attr.toString()}' ` + - "(most likely due to a circular import)"; - throw new Sk.builtin.AttributeError(msg); - } - }, init$dict(name, doc) { this.$d.__name__ = name; this.$d.__doc__ = doc; @@ -108,8 +97,12 @@ Sk.builtin.module = Sk.abstr.buildNativeClass("module", { this.$d.__loader__ = Sk.builtin.none.none$; }, sk$attrError() { - const name = this.get$name(); - return name === undefined ? "module" : "module " + name; + let name = this.get$name(); + name = name === undefined ? "module" : "module " + name; + if (this.$initializing) { + name = "(most likely due to a circular import) partially initialized " + name; + } + return name; }, get$name() { const name = this.tp$getattr(Sk.builtin.str.$name); From 3b3c1f0120e114eb9be5e4f10582f79b17fde02c Mon Sep 17 00:00:00 2001 From: stu Date: Thu, 24 Mar 2022 18:38:23 +0800 Subject: [PATCH 006/137] Circular imports - support modules that supsend --- src/import.js | 48 +++++++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/src/import.js b/src/import.js index 43992717e0..58f00f3854 100644 --- a/src/import.js +++ b/src/import.js @@ -283,32 +283,38 @@ Sk.importModuleInternal_ = function (name, dumpJS, modname, suppliedPyBody, rela module["$d"]["__file__"] = new Sk.builtin.str(co.filename); } const packageInitializing = relativeToPackage && relativeToPackage.$initializing; - module.$initializing = true; - if (relativeToPackage && !packageInitializing) { - relativeToPackage.$initializing = true; - } - try { - return modscope(module["$d"]); - } catch (e) { - try { - // don't cache a module if it raised an exception on load - Sk.abstr.objectDelItem(Sk.sysmodules, pyModName); - } catch {} - if (relativeToPackage) { + const setInitializing = (v) => { + module.$initializing = v; + if (relativeToPackage && !packageInitializing) { + relativeToPackage.$initializing = v; + } + }; + + setInitializing(true); + // our module might suspend so we need to wrap it in a suspendable try catch + // we can only setInitializing to false once the suspensions have completed + return Sk.misceval.tryCatch( + () => + Sk.misceval.chain(modscope(module["$d"]), (rv) => { + setInitializing(false); + return rv; + }), + (e) => { try { - relativeToPackage.tp$setattr(pyAttrName, undefined); + // don't cache a module if it raised an exception on load + Sk.abstr.objectDelItem(Sk.sysmodules, pyModName); } catch {} + if (relativeToPackage) { + try { + relativeToPackage.tp$setattr(pyAttrName, undefined); + } catch {} + } + setInitializing(false); + throw e; } - throw e; - } finally { - module.$initializing = false; - if (relativeToPackage && !packageInitializing) { - relativeToPackage.$initializing = false; - } - } + ); }, function (modlocs) { var i; - if (modlocs === undefined) { if (returnUndefinedOnTopLevelNotFound && !topLevelModuleToReturn) { return undefined; From 0b37949be399dec839e954188cb3f73c17fca0e5 Mon Sep 17 00:00:00 2001 From: stu Date: Thu, 24 Mar 2022 19:50:35 +0800 Subject: [PATCH 007/137] Fix nesting import chaining --- src/import.js | 6 +++--- test/unit3/test_import/__init__.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/import.js b/src/import.js index 58f00f3854..df4af0d851 100644 --- a/src/import.js +++ b/src/import.js @@ -495,7 +495,7 @@ Sk.builtin.__import__ = function (name, globals, locals, fromlist, level) { var i; var fromName; var leafModule; - const chainedImports = [null]; + var importChain; leafModule = Sk.sysmodules.mp$subscript( new Sk.builtin.str((relativeToPackageName || "") + @@ -508,13 +508,13 @@ Sk.builtin.__import__ = function (name, globals, locals, fromlist, level) { // "ret" is the module we're importing from // Only import from file system if we have not found the fromName in the current module if (fromName != "*" && leafModule.tp$getattr(new Sk.builtin.str(fromName)) === undefined) { - chainedImports.push(() => + importChain = Sk.misceval.chain(importChain, () => Sk.importModuleInternal_(fromName, undefined, undefined, undefined, leafModule, true, true) ); } } - return Sk.misceval.chain(...chainedImports, function () { + return Sk.misceval.chain(importChain, function() { // if there's a fromlist we want to return the leaf module // (ret), not the toplevel namespace Sk.asserts.assert(leafModule); diff --git a/test/unit3/test_import/__init__.py b/test/unit3/test_import/__init__.py index 7b6b6fe770..5eaaf326a9 100644 --- a/test/unit3/test_import/__init__.py +++ b/test/unit3/test_import/__init__.py @@ -69,7 +69,6 @@ def test_circular_from_import(self): "partially initialized module ", str(cm.exception) ) - print(cm.exception) self.assertIn(".circular_imports.from_cycle1'", str(cm.exception)) self.assertIn("(most likely due to a circular import)", str(cm.exception)) From 7bb9a945ed3f6acaba04f0773633a2695ebc61eb Mon Sep 17 00:00:00 2001 From: stu Date: Thu, 24 Mar 2022 21:05:21 +0800 Subject: [PATCH 008/137] fix - scoping of chained imports --- src/import.js | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/import.js b/src/import.js index df4af0d851..9c51a0f5a1 100644 --- a/src/import.js +++ b/src/import.js @@ -492,29 +492,26 @@ Sk.builtin.__import__ = function (name, globals, locals, fromlist, level) { } else { // try to load from-names as modules from the file system // if they are not present on the module itself - var i; - var fromName; - var leafModule; - var importChain; + const chainedImports = [null]; - leafModule = Sk.sysmodules.mp$subscript( + const leafModule = Sk.sysmodules.mp$subscript( new Sk.builtin.str((relativeToPackageName || "") + ((relativeToPackageName && name) ? "." : "") + name)); - for (i = 0; i < fromlist.length; i++) { - fromName = fromlist[i]; + for (let i = 0; i < fromlist.length; i++) { + const fromName = fromlist[i]; // "ret" is the module we're importing from // Only import from file system if we have not found the fromName in the current module - if (fromName != "*" && leafModule.tp$getattr(new Sk.builtin.str(fromName)) === undefined) { - importChain = Sk.misceval.chain(importChain, () => + if (fromName !== "*" && leafModule.tp$getattr(new Sk.builtin.str(fromName)) === undefined) { + chainedImports.push(() => Sk.importModuleInternal_(fromName, undefined, undefined, undefined, leafModule, true, true) ); } } - return Sk.misceval.chain(importChain, function() { + return Sk.misceval.chain(...chainedImports, function() { // if there's a fromlist we want to return the leaf module // (ret), not the toplevel namespace Sk.asserts.assert(leafModule); From 8fd21da6b4d86eae4106a8da8d308345de936c42 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Mar 2022 08:41:17 +0000 Subject: [PATCH 009/137] build(deps): bump minimist from 1.2.5 to 1.2.6 Bumps [minimist](https://github.com/substack/minimist) from 1.2.5 to 1.2.6. - [Release notes](https://github.com/substack/minimist/releases) - [Commits](https://github.com/substack/minimist/compare/1.2.5...1.2.6) --- updated-dependencies: - dependency-name: minimist dependency-type: indirect ... Signed-off-by: dependabot[bot] --- package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index e824a02fc8..c325498fab 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4040,9 +4040,9 @@ } }, "node_modules/minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==", "dev": true }, "node_modules/mississippi": { @@ -10130,9 +10130,9 @@ } }, "minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==", "dev": true }, "mississippi": { From c43afefa854ff35eabcf45c4212f9529b7d14836 Mon Sep 17 00:00:00 2001 From: Stu Cork Date: Mon, 3 May 2021 00:35:14 +0000 Subject: [PATCH 010/137] implement __slots__ with tests --- src/constants.js | 1 + src/module.js | 1 + src/object.js | 8 +- src/type.js | 111 +++++++-- test/unit/test_deepcopy.py | 33 +-- test/unit3/test_descr.py | 472 +++++++++++++++++++------------------ 6 files changed, 356 insertions(+), 270 deletions(-) diff --git a/src/constants.js b/src/constants.js index 5ca5c4900f..1da733e80d 100644 --- a/src/constants.js +++ b/src/constants.js @@ -58,6 +58,7 @@ Sk.builtin.str.$reversed = new Sk.builtin.str("__reversed__"); Sk.builtin.str.$round = new Sk.builtin.str("__round__"); Sk.builtin.str.$setattr = new Sk.builtin.str("__setattr__"); Sk.builtin.str.$setitem = new Sk.builtin.str("__setitem__"); +Sk.builtin.str.$slots = new Sk.builtin.str("__slots__"); Sk.builtin.str.$str = new Sk.builtin.str("__str__"); Sk.builtin.str.$trunc = new Sk.builtin.str("__trunc__"); Sk.builtin.str.$write = new Sk.builtin.str("write"); diff --git a/src/module.js b/src/module.js index e740b4d839..22099c1d12 100644 --- a/src/module.js +++ b/src/module.js @@ -89,6 +89,7 @@ Sk.builtin.module = Sk.abstr.buildNativeClass("module", { }, }, proto: { + sk$hasDict: true, // for reassigning __class__ init$dict(name, doc) { this.$d.__name__ = name; this.$d.__doc__ = doc; diff --git a/src/object.js b/src/object.js index 31ab861a13..b370da5c0e 100644 --- a/src/object.js +++ b/src/object.js @@ -98,7 +98,11 @@ Sk.builtin.object = Sk.abstr.buildNativeClass("object", { (oldto.sk$klass === undefined || newto.sk$klass === undefined) ) { throw new Sk.builtin.TypeError(" __class__ assignment only supported for heap types or ModuleType subclasses"); - } else if (value.prototype.sk$builtinBase !== this.sk$builtinBase) { + } else if ( + value.prototype.sk$builtinBase !== this.sk$builtinBase || + value.prototype.sk$slotsBase !== this.sk$slotsBase || + value.prototype.sk$hasDict !== this.sk$hasDict + ) { throw new Sk.builtin.TypeError( "__class__ assignment: '" + Sk.abstr.typeName(this) + "' object layout differs from '" + value.prototype.tp$name + "'" ); @@ -147,7 +151,7 @@ Sk.builtin.object = Sk.abstr.buildNativeClass("object", { }, proto: /**@lends {Sk.builtin.object.prototype}*/ { valueOf: Object.prototype.valueOf, - toString: function() { + toString() { return this.tp$str().v; }, hasOwnProperty: Object.prototype.hasOwnProperty, diff --git a/src/type.js b/src/type.js index 197d87d753..0ebe0ccaf5 100644 --- a/src/type.js +++ b/src/type.js @@ -109,20 +109,19 @@ function tp$new(args, kwargs) { throw new Sk.builtin.TypeError("type() takes 1 or 3 arguments"); } - let $name, bases, dict; - $name = args[0]; - bases = args[1]; - dict = args[2]; + const name = args[0]; + let bases = args[1]; + const dict = args[2]; // first check that we only have 3 args and they're of the correct type // argument dict must be of type dict if (dict.tp$name !== "dict") { throw new Sk.builtin.TypeError("type() argument 3 must be dict, not " + Sk.abstr.typeName(dict)); } // checks if name must be string - if (!Sk.builtin.checkString($name)) { - throw new Sk.builtin.TypeError("type() argument 1 must be str, not " + Sk.abstr.typeName($name)); + if (!Sk.builtin.checkString(name)) { + throw new Sk.builtin.TypeError("type() argument 1 must be str, not " + Sk.abstr.typeName(name)); } - $name = $name.$jsstr(); + const $name = name.$jsstr(); // argument bases must be of type tuple if (bases.tp$name !== "tuple") { throw new Sk.builtin.TypeError("type() argument 2 must be tuple, not " + Sk.abstr.typeName(bases)); @@ -133,9 +132,11 @@ function tp$new(args, kwargs) { * @type {!typeObject} */ const klass = function () { - // klass is essentially a function that gives its instances a dict - // if we support slots then we might need to have two versions of this - this.$d = new Sk.builtin.dict(); + // klass is essentially a function that gives its instances a dict and slots + if (this.sk$hasDict) { + this.$d = new Sk.builtin.dict(); + } + this.$s = {}; }; setUpKlass($name, klass, bases, this.constructor); @@ -145,18 +146,86 @@ function tp$new(args, kwargs) { } klass.prototype.__doc__ = Sk.builtin.none.none$; - - - // set __dict__ if not already on the prototype - /**@todo __slots__ */ - if (klass.$typeLookup(Sk.builtin.str.$dict) === undefined) { + let slots = dict.quick$lookup(Sk.builtin.str.$slots); + let addDict = false; + let slotSet = false; + if (slots !== undefined) { + if (Sk.builtin.checkString(slots)) { + slots = [slots]; + } else { + slots = Sk.misceval.arrayFromIterable(slots); + } + let num_slots = 0; + slotSet = new Set(); + slots.forEach((slot) => { + // check string and check slot is identifier + if (!Sk.builtin.checkString(slot)) { + throw new Sk.builtin.TypeError("__slots__ items must be strings, not '" + Sk.abstr.typeName(slot) + "'"); + } else if (!slot.$isIdentifier()) { + throw new Sk.builtin.TypeError("__slots__ must be identifiers"); + } + if (slot === Sk.builtin.str.$dict) { + addDict = true; + } else { + num_slots++; + slot = Sk.mangleName(name, slot); + const mangled = slot.$mangled; + slotSet.add(mangled); + const s = Symbol(mangled); + klass.prototype[mangled] = new Sk.builtin.getset_descriptor(klass, { + $get() { + const ret = this.$s[s]; + if (ret === undefined) { + throw new Sk.builtin.AttributeError(slot.$jsstr()); + } + return ret; + }, + $set(v) { + this.$s[s] = v; + }, + }); + } + }); + if (num_slots) { + Object.defineProperty(klass, "sk$slots", { + value: true, + writable: true, + }); + Object.defineProperty(klass.prototype, "sk$slotsBase", { + value: klass, + writable: true, + }); + } + } else if (klass.$typeLookup(Sk.builtin.str.$dict) === undefined) { + addDict = true; + } + if (addDict) { + // we only add the __dict__ descriptor if we defined it in the __slots__ + // or if we don't already have one in a superclass klass.prototype.__dict__ = new Sk.builtin.getset_descriptor(klass, subtype_dict_getset_description); } + if (addDict || klass.$typeLookup(Sk.builtin.str.$dict) !== undefined) { + Object.defineProperty(klass.prototype, "sk$hasDict", { + value: true, + writable: true, + }); + } + + if (!slotSet) { + // copy properties from dict into klass.prototype + dict.$items().forEach(([key, val]) => { + klass.prototype[key.$mangled] = val; + }); + } else { + dict.$items().forEach(([key, val]) => { + const mangled = key.$mangled; + if (slotSet.has(mangled)) { + throw new Sk.builtin.ValueError("'" + key.$jsstr() + "' in __slots__ conflicts with class variable"); + } + klass.prototype[key.$mangled] = val; + }); + } - // copy properties from dict into klass.prototype - dict.$items().forEach(([key, val]) => { - klass.prototype[key.$mangled] = val; - }); /* Set ht_qualname to dict['__qualname__'] if available, else to __name__. The __qualname__ accessor will look for ht_qualname. */ @@ -167,7 +236,7 @@ function tp$new(args, kwargs) { } klass.prototype.ht$qualname = qualname; } - + // make __new__ a static method if (klass.prototype.hasOwnProperty("__new__")) { const newf = klass.prototype.__new__; @@ -360,6 +429,8 @@ function best_base_(bases) { // if we support slots we would need to change this function - for now it just checks for the builtin. if (type.sk$solidBase) { return type; + } else if (type.sk$slots) { + return type; } return solid_base(type.prototype.tp$base); } diff --git a/test/unit/test_deepcopy.py b/test/unit/test_deepcopy.py index 4700a45a1c..b78256a348 100644 --- a/test/unit/test_deepcopy.py +++ b/test/unit/test_deepcopy.py @@ -488,22 +488,23 @@ class C(object): # self.assertIsNot(x, y) # self.assertIsNot(x["foo"], y["foo"]) - def test_copy_slots(self): - class C: - __slots__ = ["foo"] - x = C() - x.foo = [42] - y = copy.copy(x) - self.assertIs(x.foo, y.foo) - - def test_deepcopy_slots(self): - class C(object): - __slots__ = ["foo"] - x = C() - x.foo = [42] - y = copy.deepcopy(x) - self.assertEqual(x.foo, y.foo) - self.assertIsNot(x.foo, y.foo) + # todo make slots work in copy module + # def test_copy_slots(self): + # class C: + # __slots__ = ["foo"] + # x = C() + # x.foo = [42] + # y = copy.copy(x) + # self.assertIs(x.foo, y.foo) + + # def test_deepcopy_slots(self): + # class C(object): + # __slots__ = ["foo"] + # x = C() + # x.foo = [42] + # y = copy.deepcopy(x) + # self.assertEqual(x.foo, y.foo) + # self.assertIsNot(x.foo, y.foo) # def test_deepcopy_dict_subclass(self): # class C(dict): diff --git a/test/unit3/test_descr.py b/test/unit3/test_descr.py index 177c87dab3..ef36238ba1 100644 --- a/test/unit3/test_descr.py +++ b/test/unit3/test_descr.py @@ -1186,10 +1186,10 @@ class MyStr(str): with self.assertRaises(TypeError): "a".__class__ = MyStr - # class MyBytes(bytes): - # __slots__ = () - # with self.assertRaises(TypeError): - # b"a".__class__ = MyBytes + class MyBytes(bytes): + __slots__ = () + with self.assertRaises(TypeError): + b"a".__class__ = MyBytes class MyTuple(tuple): __slots__ = () @@ -1201,130 +1201,130 @@ class MyFrozenSet(frozenset): with self.assertRaises(TypeError): frozenset().__class__ = MyFrozenSet - # def test_slots(self): - # # Testing __slots__... - # class C0(object): - # __slots__ = [] - # x = C0() - # self.assertNotHasAttr(x, "__dict__") - # self.assertNotHasAttr(x, "foo") - - # class C1(object): - # __slots__ = ['a'] - # x = C1() - # self.assertNotHasAttr(x, "__dict__") - # self.assertNotHasAttr(x, "a") - # x.a = 1 - # self.assertEqual(x.a, 1) - # x.a = None - # self.assertEqual(x.a, None) - # del x.a - # self.assertNotHasAttr(x, "a") - - # class C3(object): - # __slots__ = ['a', 'b', 'c'] - # x = C3() - # self.assertNotHasAttr(x, "__dict__") - # self.assertNotHasAttr(x, 'a') - # self.assertNotHasAttr(x, 'b') - # self.assertNotHasAttr(x, 'c') - # x.a = 1 - # x.b = 2 - # x.c = 3 - # self.assertEqual(x.a, 1) - # self.assertEqual(x.b, 2) - # self.assertEqual(x.c, 3) - - # class C4(object): - # """Validate name mangling""" - # __slots__ = ['__a'] - # def __init__(self, value): - # self.__a = value - # def get(self): - # return self.__a - # x = C4(5) - # self.assertNotHasAttr(x, '__dict__') - # self.assertNotHasAttr(x, '__a') - # self.assertEqual(x.get(), 5) - # try: - # x.__a = 6 - # except AttributeError: - # pass - # else: - # self.fail("Double underscored names not mangled") + def test_slots(self): + # Testing __slots__... + class C0(object): + __slots__ = [] + x = C0() + self.assertNotHasAttr(x, "__dict__") + self.assertNotHasAttr(x, "foo") + + class C1(object): + __slots__ = ['a'] + x = C1() + self.assertNotHasAttr(x, "__dict__") + self.assertNotHasAttr(x, "a") + x.a = 1 + self.assertEqual(x.a, 1) + x.a = None + self.assertEqual(x.a, None) + del x.a + self.assertNotHasAttr(x, "a") + + class C3(object): + __slots__ = ['a', 'b', 'c'] + x = C3() + self.assertNotHasAttr(x, "__dict__") + self.assertNotHasAttr(x, 'a') + self.assertNotHasAttr(x, 'b') + self.assertNotHasAttr(x, 'c') + x.a = 1 + x.b = 2 + x.c = 3 + self.assertEqual(x.a, 1) + self.assertEqual(x.b, 2) + self.assertEqual(x.c, 3) + + class C4(object): + """Validate name mangling""" + __slots__ = ['__a'] + def __init__(self, value): + self.__a = value + def get(self): + return self.__a + x = C4(5) + self.assertNotHasAttr(x, '__dict__') + self.assertNotHasAttr(x, '__a') + self.assertEqual(x.get(), 5) + try: + x.__a = 6 + except AttributeError: + pass + else: + self.fail("Double underscored names not mangled") - # # Make sure slot names are proper identifiers - # try: - # class C(object): - # __slots__ = [None] - # except TypeError: - # pass - # else: - # self.fail("[None] slots not caught") - # try: - # class C(object): - # __slots__ = ["foo bar"] - # except TypeError: - # pass - # else: - # self.fail("['foo bar'] slots not caught") - # try: - # class C(object): - # __slots__ = ["foo\0bar"] - # except TypeError: - # pass - # else: - # self.fail("['foo\\0bar'] slots not caught") - # try: - # class C(object): - # __slots__ = ["1"] - # except TypeError: - # pass - # else: - # self.fail("['1'] slots not caught") - # try: - # class C(object): - # __slots__ = [""] - # except TypeError: - # pass - # else: - # self.fail("[''] slots not caught") - # class C(object): - # __slots__ = ["a", "a_b", "_a", "A0123456789Z"] - # # XXX(nnorwitz): was there supposed to be something tested - # # from the class above? + # Make sure slot names are proper identifiers + try: + class C(object): + __slots__ = [None] + except TypeError: + pass + else: + self.fail("[None] slots not caught") + try: + class C(object): + __slots__ = ["foo bar"] + except TypeError: + pass + else: + self.fail("['foo bar'] slots not caught") + try: + class C(object): + __slots__ = ["foo\0bar"] + except TypeError: + pass + else: + self.fail("['foo\\0bar'] slots not caught") + try: + class C(object): + __slots__ = ["1"] + except TypeError: + pass + else: + self.fail("['1'] slots not caught") + try: + class C(object): + __slots__ = [""] + except TypeError: + pass + else: + self.fail("[''] slots not caught") + class C(object): + __slots__ = ["a", "a_b", "_a", "A0123456789Z"] + # XXX(nnorwitz): was there supposed to be something tested + # from the class above? - # # Test a single string is not expanded as a sequence. - # class C(object): - # __slots__ = "abc" - # c = C() - # c.abc = 5 - # self.assertEqual(c.abc, 5) + # Test a single string is not expanded as a sequence. + class C(object): + __slots__ = "abc" + c = C() + c.abc = 5 + self.assertEqual(c.abc, 5) - # # Test unicode slot names - # # Test a single unicode string is not expanded as a sequence. - # class C(object): - # __slots__ = "abc" - # c = C() - # c.abc = 5 - # self.assertEqual(c.abc, 5) + # Test unicode slot names + # Test a single unicode string is not expanded as a sequence. + class C(object): + __slots__ = "abc" + c = C() + c.abc = 5 + self.assertEqual(c.abc, 5) - # # _unicode_to_string used to modify slots in certain circumstances - # slots = ("foo", "bar") - # class C(object): - # __slots__ = slots - # x = C() - # x.foo = 5 - # self.assertEqual(x.foo, 5) - # self.assertIs(type(slots[0]), str) - # # this used to leak references - # try: - # class C(object): - # __slots__ = [chr(128)] - # except (TypeError, UnicodeEncodeError): - # pass - # else: - # self.fail("[chr(128)] slots not caught") + # _unicode_to_string used to modify slots in certain circumstances + slots = ("foo", "bar") + class C(object): + __slots__ = slots + x = C() + x.foo = 5 + self.assertEqual(x.foo, 5) + self.assertIs(type(slots[0]), str) + # this used to leak references + try: + class C(object): + __slots__ = [chr(128)] + except (TypeError, UnicodeEncodeError): + pass + else: + self.fail("[chr(128)] slots not caught") # # Test leaks # class Counted(object): @@ -1403,43 +1403,43 @@ class MyFrozenSet(frozenset): # with self.assertRaises(AttributeError): # del X().a - # def test_slots_special(self): - # # Testing __dict__ and __weakref__ in __slots__... - # class D(object): - # __slots__ = ["__dict__"] - # a = D() - # self.assertHasAttr(a, "__dict__") - # self.assertNotHasAttr(a, "__weakref__") - # a.foo = 42 - # self.assertEqual(a.__dict__, {"foo": 42}) - - # class W(object): - # __slots__ = ["__weakref__"] - # a = W() - # self.assertHasAttr(a, "__weakref__") - # self.assertNotHasAttr(a, "__dict__") - # try: - # a.foo = 42 - # except AttributeError: - # pass - # else: - # self.fail("shouldn't be allowed to set a.foo") - - # class C1(W, D): - # __slots__ = [] - # a = C1() - # self.assertHasAttr(a, "__dict__") - # self.assertHasAttr(a, "__weakref__") - # a.foo = 42 - # self.assertEqual(a.__dict__, {"foo": 42}) - - # class C2(D, W): - # __slots__ = [] - # a = C2() - # self.assertHasAttr(a, "__dict__") - # self.assertHasAttr(a, "__weakref__") - # a.foo = 42 - # self.assertEqual(a.__dict__, {"foo": 42}) + def test_slots_special(self): + # Testing __dict__ and __weakref__ in __slots__... + class D(object): + __slots__ = ["__dict__"] + a = D() + self.assertHasAttr(a, "__dict__") + self.assertNotHasAttr(a, "__weakref__") + a.foo = 42 + self.assertEqual(a.__dict__, {"foo": 42}) + + class W(object): + __slots__ = ["__weakref__"] + a = W() + # self.assertHasAttr(a, "__weakref__") + self.assertNotHasAttr(a, "__dict__") + try: + a.foo = 42 + except AttributeError: + pass + else: + self.fail("shouldn't be allowed to set a.foo") + + class C1(W, D): + __slots__ = [] + a = C1() + self.assertHasAttr(a, "__dict__") + # self.assertHasAttr(a, "__weakref__") + a.foo = 42 + self.assertEqual(a.__dict__, {"foo": 42}) + + class C2(D, W): + __slots__ = [] + a = C2() + self.assertHasAttr(a, "__dict__") + # self.assertHasAttr(a, "__weakref__") + a.foo = 42 + self.assertEqual(a.__dict__, {"foo": 42}) # def test_slots_special2(self): # # Testing __qualname__ and __classcell__ in __slots__ @@ -1596,21 +1596,21 @@ class C(type(len)): else: self.fail("inheritance from CFunction should be illegal") - # try: - # class C(object): - # __slots__ = 1 - # except TypeError: - # pass - # else: - # self.fail("__slots__ = 1 should be illegal") + try: + class C(object): + __slots__ = 1 + except TypeError: + pass + else: + self.fail("__slots__ = 1 should be illegal") - # try: - # class C(object): - # __slots__ = [1] - # except TypeError: - # pass - # else: - # self.fail("__slots__ = [1] should be illegal") + try: + class C(object): + __slots__ = [1] + except TypeError: + pass + else: + self.fail("__slots__ = [1] should be illegal") # class M1(type): # pass @@ -3388,45 +3388,53 @@ class Int(int): __slots__ = [] cant(o, type(1)) cant(o, type(None)) del o - # class G(object): - # __slots__ = ["a", "b"] - # class H(object): - # __slots__ = ["b", "a"] - # class I(object): - # __slots__ = ["a", "b"] - # class J(object): - # __slots__ = ["c", "b"] - # class K(object): - # __slots__ = ["a", "b", "d"] - # class L(H): - # __slots__ = ["e"] - # class M(I): - # __slots__ = ["e"] - # class N(J): - # __slots__ = ["__weakref__"] - # class P(J): - # __slots__ = ["__dict__"] - # class Q(J): - # pass - # class R(J): - # __slots__ = ["__dict__", "__weakref__"] + class G(object): + __slots__ = ["a", "b"] + class H(object): + __slots__ = ["b", "a"] + class I(object): + __slots__ = ["a", "b"] + class J(object): + __slots__ = ["c", "b"] + class K(object): + __slots__ = ["a", "b", "d"] + class L(H): + __slots__ = ["e"] + class M(I): + __slots__ = ["e"] + class N(J): + # __slots__ = ["__weakref__"] + __slots__ = [] + class P(J): + __slots__ = ["__dict__"] + class Q(J): + pass + class R(J): + # __slots__ = ["__dict__", "__weakref__"] + __slots__ = ["__dict__"] + # skulpt implementation detail means that this wont work # for cls, cls2 in ((G, H), (G, I), (I, H), (Q, R), (R, Q)): - # x = cls() - # x.a = 1 - # x.__class__ = cls2 - # self.assertIs(x.__class__, cls2, - # "assigning %r as __class__ for %r silently failed" % (cls2, x)) - # self.assertEqual(x.a, 1) - # x.__class__ = cls - # self.assertIs(x.__class__, cls, - # "assigning %r as __class__ for %r silently failed" % (cls, x)) - # self.assertEqual(x.a, 1) - # for cls in G, J, K, L, M, N, P, R, list, Int: - # for cls2 in G, J, K, L, M, N, P, R, list, Int: - # if cls is cls2: - # continue - # cant(cls(), cls2) + for cls, cls2 in ((Q, R), (R, Q)): + x = cls() + x.a = 1 + x.__class__ = cls2 + self.assertIs(x.__class__, cls2, + "assigning %r as __class__ for %r silently failed" % (cls2, x)) + self.assertEqual(x.a, 1) + x.__class__ = cls + self.assertIs(x.__class__, cls, + "assigning %r as __class__ for %r silently failed" % (cls, x)) + self.assertEqual(x.a, 1) + for cls in G, J, K, L, M, N, P, R, list, Int: + for cls2 in G, J, K, L, M, N, P, R, list, Int: + if cls is cls2: + continue + # added for skulpt since we don't do weakrefs + x = {cls, cls2} + if x & {P, Q, R} == x or x == {N, J}: + continue + cant(cls(), cls2) # # Issue5283: when __class__ changes in __del__, the wrong # # type gets DECREF'd. @@ -4001,19 +4009,19 @@ def __getitem__(self, x): # o = trash(o) # del o - # def test_slots_multiple_inheritance(self): - # # SF bug 575229, multiple inheritance w/ slots dumps core - # class A(object): - # __slots__=() - # class B(object): - # pass - # class C(A,B) : - # __slots__=() - # # if support.check_impl_detail(): - # # self.assertEqual(C.__basicsize__, B.__basicsize__) - # self.assertHasAttr(C, '__dict__') - # self.assertHasAttr(C, '__weakref__') - # C().x = 2 + def test_slots_multiple_inheritance(self): + # SF bug 575229, multiple inheritance w/ slots dumps core + class A(object): + __slots__=() + class B(object): + pass + class C(A,B) : + __slots__=() + # if support.check_impl_detail(): + # self.assertEqual(C.__basicsize__, B.__basicsize__) + self.assertHasAttr(C, '__dict__') + # self.assertHasAttr(C, '__weakref__') + C().x = 2 def test_rmul(self): # Testing correct invocation of __rmul__... @@ -4832,13 +4840,13 @@ class A(int): with self.assertRaises(TypeError): a + a - # def test_slot_shadows_class_variable(self): - # with self.assertRaises(ValueError) as cm: - # class X: - # __slots__ = ["foo"] - # foo = None - # m = str(cm.exception) - # self.assertEqual("'foo' in __slots__ conflicts with class variable", m) + def test_slot_shadows_class_variable(self): + with self.assertRaises(ValueError) as cm: + class X: + __slots__ = ["foo"] + foo = None + m = str(cm.exception) + self.assertEqual("'foo' in __slots__ conflicts with class variable", m) def test_set_doc(self): class X: From 50ab134accf85cdc9c5799ea1e89f6122c3083e2 Mon Sep 17 00:00:00 2001 From: Stu Cork Date: Sun, 10 Oct 2021 00:37:12 +0800 Subject: [PATCH 011/137] __slots__: do some tidy up and support .__class__ allocation for __slots__ if compatable --- src/object.js | 68 ++++++++++++++--- src/type.js | 156 +++++++++++++++++++-------------------- test/unit3/test_descr.py | 4 +- 3 files changed, 137 insertions(+), 91 deletions(-) diff --git a/src/object.js b/src/object.js index b370da5c0e..a20e1a0754 100644 --- a/src/object.js +++ b/src/object.js @@ -95,20 +95,12 @@ Sk.builtin.object = Sk.abstr.buildNativeClass("object", { if ( !(oldto.$isSubType(Sk.builtin.module) && newto.$isSubType(Sk.builtin.module)) && - (oldto.sk$klass === undefined || newto.sk$klass === undefined) + (oldto.prototype.ht$type === undefined || newto.prototype.ht$type === undefined) ) { throw new Sk.builtin.TypeError(" __class__ assignment only supported for heap types or ModuleType subclasses"); - } else if ( - value.prototype.sk$builtinBase !== this.sk$builtinBase || - value.prototype.sk$slotsBase !== this.sk$slotsBase || - value.prototype.sk$hasDict !== this.sk$hasDict - ) { - throw new Sk.builtin.TypeError( - "__class__ assignment: '" + Sk.abstr.typeName(this) + "' object layout differs from '" + value.prototype.tp$name + "'" - ); } + checkCompatibleForAssignment(oldto, newto); Object.setPrototypeOf(this, value.prototype); - return; }, $doc: "the object's class", }, @@ -198,3 +190,59 @@ Sk.builtin.object = Sk.abstr.buildNativeClass("object", { Sk.abstr.setUpBuiltinMro(Sk.builtin.type); })(); +function compatibleWithTpBase(child) { + const childProto = child.prototype; + const parent = childProto.tp$base; + if (parent == null) { + return false; + } + const parentProto = parent.prototype; + if (parent.sk$solidSlotBase || child.sk$solidSlotBase) { + return false; + } else if (parentProto.sk$hasDict !== childProto.sk$hasDict) { + return false; + } else if (parent.sk$solidBase && parent !== Sk.builtin.module) { + return false; + } + return true; +} + +function sameSlotsAdded(a, b) { + const aProto = a.prototype; + const bProto = b.prototype; + const aSlots = aProto.ht$slots; + const bSlots = bProto.ht$slots; + if (aProto.sk$hasDict !== bProto.sk$hasDict) { + return false; + } + if (aSlots === bSlots) { + return true; + } else if (aSlots && bSlots) { + return aSlots.length === bSlots.length && aSlots.every((s, i) => s === bSlots[i]); + } + return (aSlots && (aSlots.length || null)) === (bSlots && (bSlots.length || null)); +} + +function checkCompatibleForAssignment(newto, oldto) { + let newbase = newto; + let oldbase = oldto; + + while (compatibleWithTpBase(newbase)) { + newbase = newbase.prototype.tp$base; + } + while (compatibleWithTpBase(oldbase)) { + oldbase = oldbase.prototype.tp$base; + } + if ( + newbase !== oldbase && + (newbase.prototype.tp$base !== oldbase.prototype.tp$base || !sameSlotsAdded(newbase, oldbase)) + ) { + throw new Sk.builtin.TypeError( + "__class__ assignment: '" + + oldto.prototype.tp$name + + "' object layout differs from '" + + newto.prototype.tp$name + + "'" + ); + } +} \ No newline at end of file diff --git a/src/type.js b/src/type.js index 0ebe0ccaf5..c41e79eb38 100644 --- a/src/type.js +++ b/src/type.js @@ -135,114 +135,88 @@ function tp$new(args, kwargs) { // klass is essentially a function that gives its instances a dict and slots if (this.sk$hasDict) { this.$d = new Sk.builtin.dict(); - } - this.$s = {}; + } + // use an array for slots - slots may be added at any index; + this.$s = []; }; setUpKlass($name, klass, bases, this.constructor); + const klassProto = klass.prototype; // set some defaults which can be overridden by the dict object if (Sk.globals) { - klass.prototype.__module__ = Sk.globals["__name__"]; + klassProto.__module__ = Sk.globals["__name__"]; } - klass.prototype.__doc__ = Sk.builtin.none.none$; + klassProto.__doc__ = Sk.builtin.none.none$; + + // __slots__ + let slotNames = dict.quick$lookup(Sk.builtin.str.$slots); + let wantDict = slotNames === undefined; + let protoHasDict = klass.$typeLookup(Sk.builtin.str.$dict) !== undefined; + let slotSet; + + if (slotNames !== undefined) { + slotSet = new Set(); - let slots = dict.quick$lookup(Sk.builtin.str.$slots); - let addDict = false; - let slotSet = false; - if (slots !== undefined) { - if (Sk.builtin.checkString(slots)) { - slots = [slots]; + if (Sk.builtin.checkString(slotNames)) { + slotNames = [slotNames]; } else { - slots = Sk.misceval.arrayFromIterable(slots); + slotNames = Sk.misceval.arrayFromIterable(slotNames); } - let num_slots = 0; - slotSet = new Set(); - slots.forEach((slot) => { + + slotNames.forEach((slotName) => { // check string and check slot is identifier - if (!Sk.builtin.checkString(slot)) { - throw new Sk.builtin.TypeError("__slots__ items must be strings, not '" + Sk.abstr.typeName(slot) + "'"); - } else if (!slot.$isIdentifier()) { + if (!Sk.builtin.checkString(slotName)) { + throw new Sk.builtin.TypeError("__slots__ items must be strings, not '" + Sk.abstr.typeName(slotName) + "'"); + } else if (!slotName.$isIdentifier()) { throw new Sk.builtin.TypeError("__slots__ must be identifiers"); } - if (slot === Sk.builtin.str.$dict) { - addDict = true; + if (slotName === Sk.builtin.str.$dict) { + if (protoHasDict) { + throw new Sk.builtin.TypeError("__dict__ slot disallowed: we already got one"); + } + wantDict = true; } else { - num_slots++; - slot = Sk.mangleName(name, slot); - const mangled = slot.$mangled; - slotSet.add(mangled); - const s = Symbol(mangled); - klass.prototype[mangled] = new Sk.builtin.getset_descriptor(klass, { - $get() { - const ret = this.$s[s]; - if (ret === undefined) { - throw new Sk.builtin.AttributeError(slot.$jsstr()); - } - return ret; - }, - $set(v) { - this.$s[s] = v; - }, - }); + slotSet.add(Sk.mangleName(name, slotName)); } }); - if (num_slots) { - Object.defineProperty(klass, "sk$slots", { - value: true, - writable: true, - }); - Object.defineProperty(klass.prototype, "sk$slotsBase", { - value: klass, - writable: true, - }); - } - } else if (klass.$typeLookup(Sk.builtin.str.$dict) === undefined) { - addDict = true; + slotNames = [...slotSet].sort((a, b) => a.toString().localeCompare(b.toString())); + createSlots(slotNames, klass); } - if (addDict) { + klassProto.ht$slots = slotNames || null; // sorted Array or null + + if (wantDict && !protoHasDict) { // we only add the __dict__ descriptor if we defined it in the __slots__ - // or if we don't already have one in a superclass - klass.prototype.__dict__ = new Sk.builtin.getset_descriptor(klass, subtype_dict_getset_description); - } - if (addDict || klass.$typeLookup(Sk.builtin.str.$dict) !== undefined) { - Object.defineProperty(klass.prototype, "sk$hasDict", { - value: true, - writable: true, - }); + // or if we don't already have one on our prototype + klassProto.__dict__ = new Sk.builtin.getset_descriptor(klass, subtype_dict_getset_description); + protoHasDict = true; } + // a flag added to every heaptype prototype for quick lookup in the klass constructor + klassProto.sk$hasDict = protoHasDict; - if (!slotSet) { - // copy properties from dict into klass.prototype - dict.$items().forEach(([key, val]) => { - klass.prototype[key.$mangled] = val; - }); - } else { - dict.$items().forEach(([key, val]) => { - const mangled = key.$mangled; - if (slotSet.has(mangled)) { - throw new Sk.builtin.ValueError("'" + key.$jsstr() + "' in __slots__ conflicts with class variable"); - } - klass.prototype[key.$mangled] = val; - }); - } + dict.$items().forEach(([key, val]) => { + if (slotSet && slotSet.has(key)) { + throw new Sk.builtin.ValueError("'" + key.toString() + "' in __slots__ conflicts with class variable"); + } + klassProto[key.$mangled] = val; + }); /* Set ht_qualname to dict['__qualname__'] if available, else to __name__. The __qualname__ accessor will look for ht_qualname. */ - if (klass.prototype.hasOwnProperty("__qualname__")) { - const qualname = klass.prototype.__qualname__; + if (klassProto.hasOwnProperty("__qualname__")) { + const qualname = klassProto.__qualname__; if (!Sk.builtin.checkString(qualname)) { throw new Sk.builtin.TypeError("type __qualname__ must be a str, not '" + Sk.abstr.typeName(qualname) + "'"); } - klass.prototype.ht$qualname = qualname; + klassProto.ht$qualname = qualname; } // make __new__ a static method - if (klass.prototype.hasOwnProperty("__new__")) { - const newf = klass.prototype.__new__; + if (klassProto.hasOwnProperty("__new__")) { + const newf = klassProto.__new__; if (newf instanceof Sk.builtin.func) { // __new__ is an implied staticmethod - klass.prototype.__new__ = new Sk.builtin.staticmethod(newf); + klassProto.__new__ = new Sk.builtin.staticmethod(newf); } } klass.$allocateSlots(); @@ -429,7 +403,7 @@ function best_base_(bases) { // if we support slots we would need to change this function - for now it just checks for the builtin. if (type.sk$solidBase) { return type; - } else if (type.sk$slots) { + } else if (type.sk$solidSlotBase) { return type; } return solid_base(type.prototype.tp$base); @@ -459,6 +433,32 @@ function best_base_(bases) { return base; } +function createSlots(slotNames, klass) { + const klassProto = klass.prototype; + const nextSlotIdx = klassProto.sk$nslots || 0; + klassProto.sk$nslots = nextSlotIdx + slotNames.length; + if (slotNames.length) { + klass.sk$solidSlotBase = true; + } + + slotNames.forEach((slotName, i) => { + i += nextSlotIdx; + const mangled = slotName.$mangled; + klassProto[mangled] = new Sk.builtin.getset_descriptor(klass, { + $get() { + const ret = this.$s[i]; + if (ret === undefined) { + throw new Sk.builtin.AttributeError(slotName); + } + return ret; + }, + $set(v) { + this.$s[i] = v; + }, + }); + }); +} + function $mroMerge(seqs) { this.prototype.sk$prototypical = true; // assume true to start with let seq, i, j; diff --git a/test/unit3/test_descr.py b/test/unit3/test_descr.py index ef36238ba1..9b05550dc3 100644 --- a/test/unit3/test_descr.py +++ b/test/unit3/test_descr.py @@ -3413,9 +3413,7 @@ class R(J): # __slots__ = ["__dict__", "__weakref__"] __slots__ = ["__dict__"] - # skulpt implementation detail means that this wont work - # for cls, cls2 in ((G, H), (G, I), (I, H), (Q, R), (R, Q)): - for cls, cls2 in ((Q, R), (R, Q)): + for cls, cls2 in ((G, H), (G, I), (I, H), (Q, R), (R, Q)): x = cls() x.a = 1 x.__class__ = cls2 From 975171eb9f2abcfee3680bb16aa0a37239d1616c Mon Sep 17 00:00:00 2001 From: mrcork Date: Sat, 20 Feb 2021 10:00:06 +0800 Subject: [PATCH 012/137] metaclass proof of concept --- src/compile.js | 39 ++++++++++++++++++++++++++++++++++++--- src/misceval.js | 14 +++++++++++--- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/src/compile.js b/src/compile.js index e6185642bc..f2715ba67c 100644 --- a/src/compile.js +++ b/src/compile.js @@ -318,6 +318,38 @@ Compiler.prototype.cunpackstarstoarray = function(elts, permitEndOnly) { } }; +Compiler.prototype.cunpackkwstoarray = function (keywords, codeobj) { + + let keywordArgs = "undefined"; + + if (keywords && keywords.length > 0) { + let hasStars = false; + const kwarray = []; + for (let kw of keywords) { + if (hasStars && !Sk.__future__.python3) { + throw new SyntaxError("Advanced unpacking of function arguments is not supported in Python 2"); + } + if (kw.arg) { + kwarray.push("'" + kw.arg.v + "'"); + kwarray.push(this.vexpr(kw.value)); + } else { + hasStars = true; + } + } + keywordArgs = "[" + kwarray.join(",") + "]"; + if (hasStars) { + keywordArgs = this._gr("keywordArgs", keywordArgs); + for (let kw of keywords) { + if (!kw.arg) { + out("$ret = Sk.abstr.mappingUnpackIntoKeywordArray(",keywordArgs,",",this.vexpr(kw.value),",",codeobj,");"); + this._checkSuspension(); + } + } + } + } + return keywordArgs; +}; + Compiler.prototype.ctuplelistorset = function(e, data, tuporlist) { var i; var items; @@ -636,7 +668,6 @@ Compiler.prototype.ccompare = function (e) { Compiler.prototype.ccall = function (e) { var func = this.vexpr(e.func); - var kwarray = null; // Okay, here's the deal. We have some set of positional args // and we need to unpack them. We have some set of keyword args // and we need to unpack those too. Then we make a call. @@ -644,7 +675,7 @@ Compiler.prototype.ccall = function (e) { // help us here; we do it by hand. let positionalArgs = this.cunpackstarstoarray(e.args, !Sk.__future__.python3); - let keywordArgs = "undefined"; + let keywordArgs = this.cunpackkwstoarray(e.keywords, func); if (e.keywords && e.keywords.length > 0) { let hasStars = false; @@ -2492,6 +2523,7 @@ Compiler.prototype.cclass = function (s) { bases = this.vseqexpr(s.bases); + let keywordArgs = this.cunpackkwstoarray(s.keywords); scopename = this.enterScope(s.name, s, s.lineno); entryBlock = this.newBlock("class entry"); @@ -2522,7 +2554,8 @@ Compiler.prototype.cclass = function (s) { this.exitScope(); // todo; metaclass - out("$ret = Sk.misceval.buildClass($gbl,", scopename, ",", s.name["$r"]().v, ",[", bases, "], $cell);"); + out("$ret = Sk.misceval.buildClass($gbl,", scopename, ",", s.name["$r"]().v, ",[", bases, "], $cell, ", keywordArgs, ");"); + this._checkSuspension(); // apply decorators diff --git a/src/misceval.js b/src/misceval.js index 77ff5664e4..f28344962f 100644 --- a/src/misceval.js +++ b/src/misceval.js @@ -1323,10 +1323,18 @@ Sk.exportSymbol("Sk.misceval.promiseToSuspension", Sk.misceval.promiseToSuspensi * should return a newly constructed class object. * */ -Sk.misceval.buildClass = function (globals, func, name, bases, cell) { +Sk.misceval.buildClass = function (globals, func, name, bases, cell, kws) { // todo; metaclass var klass; - var meta = Sk.builtin.type; + kws = kws || []; + let meta_idx = kws.indexOf("metaclass"); + let meta; + if (meta_idx > -1) { + meta = kws[meta_idx + 1]; + kws.splice(meta_idx, 2); + } else { + meta = Sk.builtin.type; + } var l_cell = cell === undefined ? {} : cell; var locals = {}; @@ -1358,7 +1366,7 @@ Sk.misceval.buildClass = function (globals, func, name, bases, cell) { } _locals = new Sk.builtin.dict(_locals); - klass = Sk.misceval.callsimArray(meta, [_name, _bases, _locals]); + klass = Sk.misceval.callsimOrSuspendArray(meta, [_name, _bases, _locals], kws); return klass; }; From 0ca2a2951f8df73332e41327891ce02c3a0f0cb7 Mon Sep 17 00:00:00 2001 From: mrcork Date: Sat, 20 Feb 2021 10:00:06 +0800 Subject: [PATCH 013/137] metaclasses, init_sucblass and set_names --- src/compile.js | 9 +- src/constants.js | 3 + src/descr.js | 2 +- src/errors.js | 11 + src/lib/document.js | 2 - src/main.js | 1 + src/misceval.js | 99 ++- src/object.js | 8 + src/type.js | 52 +- test/run/t407.py.real | 4 +- test/run/t407.py.real.force | 4 +- test/unit3/test_descr.py | 995 ++++++++++++++----------------- test/unit3/test_init_subclass.py | 301 ++++++++++ test/unit3/test_super.py | 3 +- 14 files changed, 921 insertions(+), 573 deletions(-) create mode 100644 test/unit3/test_init_subclass.py diff --git a/src/compile.js b/src/compile.js index f2715ba67c..3d2e78efa2 100644 --- a/src/compile.js +++ b/src/compile.js @@ -708,8 +708,9 @@ Compiler.prototype.ccall = function (e) { // note that it's part of the js API spec: https://developer.mozilla.org/en/docs/Web/API/Window/self // so we should probably add self to the mangling // TODO: feel free to ignore the above - out("if (typeof self === \"undefined\" || self === Sk.global) { throw new Sk.builtin.RuntimeError(\"super(): no arguments\") };"); - positionalArgs = "[$gbl.__class__,self]"; + this.u.tempsToSave.push("$sup"); + out("if (typeof $sup === \"undefined\") { throw new Sk.builtin.RuntimeError(\"super(): no arguments\") };"); + positionalArgs = "[$gbl.__class__,$sup]"; } out ("$ret = (",func,".tp$call)?",func,".tp$call(",positionalArgs,",",keywordArgs,") : Sk.misceval.applyOrSuspend(",func,",undefined,undefined,",keywordArgs,",",positionalArgs,");"); @@ -2090,8 +2091,10 @@ Compiler.prototype.buildcodeobj = function (n, coname, decorator_list, args, cal } else { this.u.varDeclsCode += "\nvar $args = this.$resolveArgs($posargs,$kwargs)\n"; } + const sup_i = kwarg ? 1 : 0; for (let i=0; i < funcArgs.length; i++) { - this.u.varDeclsCode += ","+funcArgs[i]+"=$args["+i+"]"; + const sup = i === sup_i ? "$sup = " : "" + this.u.varDeclsCode += ","+sup+funcArgs[i]+"=$args["+i+"]"; } this.u.varDeclsCode += ";\n"; } diff --git a/src/constants.js b/src/constants.js index 1da733e80d..2effb22104 100644 --- a/src/constants.js +++ b/src/constants.js @@ -29,6 +29,7 @@ Sk.builtin.str.$eq = new Sk.builtin.str("__eq__"); Sk.builtin.str.$exit = new Sk.builtin.str("__exit__"); Sk.builtin.str.$index = new Sk.builtin.str("__index__"); Sk.builtin.str.$init = new Sk.builtin.str("__init__"); +Sk.builtin.str.$initsubclass = new Sk.builtin.str("__init_subclass__"); Sk.builtin.str.$int_ = new Sk.builtin.str("__int__"); Sk.builtin.str.$iter = new Sk.builtin.str("__iter__"); Sk.builtin.str.$file = new Sk.builtin.str("__file__"); @@ -52,6 +53,7 @@ Sk.builtin.str.$ne = new Sk.builtin.str("__ne__"); Sk.builtin.str.$new = new Sk.builtin.str("__new__"); Sk.builtin.str.$next = new Sk.builtin.str("__next__"); Sk.builtin.str.$path = new Sk.builtin.str("__path__"); +Sk.builtin.str.$prepare = new Sk.builtin.str("__prepare__"); Sk.builtin.str.$qualname = new Sk.builtin.str("__qualname__"); Sk.builtin.str.$repr = new Sk.builtin.str("__repr__"); Sk.builtin.str.$reversed = new Sk.builtin.str("__reversed__"); @@ -60,6 +62,7 @@ Sk.builtin.str.$setattr = new Sk.builtin.str("__setattr__"); Sk.builtin.str.$setitem = new Sk.builtin.str("__setitem__"); Sk.builtin.str.$slots = new Sk.builtin.str("__slots__"); Sk.builtin.str.$str = new Sk.builtin.str("__str__"); +Sk.builtin.str.$setname = new Sk.builtin.str("__set_name__"); Sk.builtin.str.$trunc = new Sk.builtin.str("__trunc__"); Sk.builtin.str.$write = new Sk.builtin.str("write"); diff --git a/src/descr.js b/src/descr.js index 18627a7b3b..5323888499 100644 --- a/src/descr.js +++ b/src/descr.js @@ -354,7 +354,7 @@ Sk.builtin.classmethod_descriptor = buildDescriptor("classmethod_descriptor", "m ); } } - if (type.ob$type !== Sk.builtin.type) { + if (!type.ob$type.$isSubType(Sk.builtin.type)) { throw new Sk.builtin.TypeError( "descriptor '" + this.d$name + diff --git a/src/errors.js b/src/errors.js index e128b94d17..c3befd1f9b 100644 --- a/src/errors.js +++ b/src/errors.js @@ -49,6 +49,17 @@ const BaseException = Sk.abstr.buildNativeClass("BaseException", { this.args = new Sk.builtin.tuple(v); }, }, + __cause__ : { + $get() { + return this.$cause || Sk.builtin.none.none$; + }, + $set(v) { + if (!Sk.builtin.checkNone(v) && !(v instanceof Sk.builtin.BaseException)) { + throw new TypeError("exception cause must be None or derive from BaseException"); + } + this.$cause = v; + } + }, __dict__: Sk.generic.getSetDict, /**@todo */ // __traceback__: {}, diff --git a/src/lib/document.js b/src/lib/document.js index 03637b4577..440089d962 100644 --- a/src/lib/document.js +++ b/src/lib/document.js @@ -86,8 +86,6 @@ var $builtinmodule = function (name) { }) - $loc.tp$getattr = Sk.generic.getAttr; - $loc.__setattr__ = new Sk.builtin.func(function (self, key, value) { key = Sk.ffi.remapToJs(key); if (key === 'innerHTML') { diff --git a/src/main.js b/src/main.js index d738da3c87..82f5001ded 100644 --- a/src/main.js +++ b/src/main.js @@ -29,6 +29,7 @@ require("./sk_method.js"); Sk.abstr.setUpSlots(cls); Sk.abstr.setUpMethods(cls); Sk.abstr.setUpGetSets(cls); + Sk.abstr.setUpClassMethods(cls); }); require("./nonetype"); require("./formatting.js"); diff --git a/src/misceval.js b/src/misceval.js index f28344962f..2fe6aa4dd5 100644 --- a/src/misceval.js +++ b/src/misceval.js @@ -1324,50 +1324,99 @@ Sk.exportSymbol("Sk.misceval.promiseToSuspension", Sk.misceval.promiseToSuspensi * */ Sk.misceval.buildClass = function (globals, func, name, bases, cell, kws) { - // todo; metaclass - var klass; + const _name = new Sk.builtin.str(name); + const _bases = update_bases(bases); // todo this function should go through the bases and check for __mro_entries__ + kws = kws || []; - let meta_idx = kws.indexOf("metaclass"); let meta; + let is_class = true; + const meta_idx = kws.indexOf("metaclass"); if (meta_idx > -1) { meta = kws[meta_idx + 1]; kws.splice(meta_idx, 2); + is_class = Sk.builtin.checkClass(meta); } else { - meta = Sk.builtin.type; + if (!bases.length) { + /* if there are no bases, use type: */ + meta = Sk.builtin.type; + } else { + /* else get the type of the first base */ + meta = bases[0].ob$type; + } + } + + if (is_class) { + meta = calculate_meta(meta, bases); // we should do this in type as well + } + /* else: meta is not a class, so we cannot do the metaclass + calculation, so we will use the explicitly given object as it is */ + + let ns; + if (meta === Sk.builtin.type) { + ns = null; // fast path + } else { + const prep = meta.tp$getattr(Sk.builtin.str.$prepare); + if (prep === undefined) { + ns = null; + } else { + ns = Sk.misceval.callsimArray(prep, [_name, _bases], kws); + if (!Sk.builtin.checkMapping(ns)) { + // in python this can be any mapping + throw new Sk.builtin.TypeError(is_class ? meta.prototype.tp$name : "" + ".__prepare__() must be a mapping not '" + Sk.abstr.typeName(ns) + "'"); + } + } } - var l_cell = cell === undefined ? {} : cell; - var locals = {}; + const l_cell = cell === undefined ? {} : cell; + const locals = {}; + if (ns === null) { + ns = new Sk.builtin.dict([]); + } else { + // metaclass is not type and __prepare__ could have returned any mapping + const keys = Sk.abstr.iter(Sk.misceval.callsimArray(ns.tp$getattr(Sk.builtin.str.$keys))); + for (let key = keys.tp$iternext(); key !== undefined; key = keys.tp$iternext()) { + if (Sk.builtin.checkString(key)) { + locals[key.toString()] = ns.mp$subscript(key); // ignore non strings + } + } + } // init the dict for the class func(globals, locals, l_cell); - // ToDo: check if func contains the __meta__ attribute - // or if the bases contain __meta__ - // new Syntax would be different // file's __name__ is class's __module__ if (globals["__name__"]) { // some js modules haven't set their module name and we shouldn't set a dictionary value to be undefined that should be equivalent to deleting a value; locals.__module__ = globals["__name__"]; } - var _name = new Sk.builtin.str(name); - var _bases = new Sk.builtin.tuple(bases); - var _locals = []; - var key; - - // build array for python dict - for (key in locals) { - if (!locals.hasOwnProperty(key)) { - //The current property key not a direct property of p - continue; - } - _locals.push(new Sk.builtin.str(key)); // push key - _locals.push(locals[key]); // push associated value - } - _locals = new Sk.builtin.dict(_locals); - klass = Sk.misceval.callsimOrSuspendArray(meta, [_name, _bases, _locals], kws); + Object.keys(locals).forEach((key) => { + ns.mp$ass_subscript(new Sk.builtin.str(key), locals[key]); + }); + + const klass = Sk.misceval.callsimOrSuspendArray(meta, [_name, _bases, ns], kws); return klass; }; Sk.exportSymbol("Sk.misceval.buildClass", Sk.misceval.buildClass); + + +function update_bases(bases) { + // todo + return new Sk.builtin.tuple(bases); +} + +function calculate_meta(meta, bases) { + let winner = meta; + bases.forEach(base => { + let tmptype = base.ob$type; + if (winner.$isSubType(tmptype)) { + // continue + } else if (tmptype.$isSubType(winner)) { + winner = tmptype; + } else { + throw new Sk.builtin.TypeError("metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases"); + } + }); + return winner; +} \ No newline at end of file diff --git a/src/object.js b/src/object.js index a20e1a0754..9edce5909c 100644 --- a/src/object.js +++ b/src/object.js @@ -141,6 +141,14 @@ Sk.builtin.object = Sk.abstr.buildNativeClass("object", { $doc: "Default object formatter.", }, }, + classmethods: { + __init_subclass__: { + $meth(args) { + return Sk.builtin.none.none$; + }, + $flags: { FastCall: true, NoKwargs: true }, + }, + }, proto: /**@lends {Sk.builtin.object.prototype}*/ { valueOf: Object.prototype.valueOf, toString() { diff --git a/src/type.js b/src/type.js index c41e79eb38..f584a7cb78 100644 --- a/src/type.js +++ b/src/type.js @@ -34,6 +34,7 @@ Object.defineProperties( tp$doc: "type(object_or_name, bases, dict)\ntype(object) -> the object's type\ntype(name, bases, dict) -> a new type", tp$call, tp$new, + tp$init, tp$getattr, tp$setattr, $r, @@ -41,6 +42,7 @@ Object.defineProperties( writable: true, }, tp$methods: { value: null, writable: true }, // define these later + tp$classmethods: { value: null, writable: true }, // define these later tp$getsets: { value: null, writable: true }, sk$type: { value: true }, $isSubType: { value: $isSubType }, @@ -211,6 +213,14 @@ function tp$new(args, kwargs) { klassProto.ht$qualname = qualname; } + // make __init_subclass__ a classmethod + if (klass.prototype.hasOwnProperty("__init_subclass__")) { + const initsubclass = klass.prototype.__init_subclass__; + if (initsubclass instanceof Sk.builtin.func) { + // initsubclass is an implied classmethod + klass.prototype.__init_subclass__ = new Sk.builtin.classmethod(initsubclass); + } + } // make __new__ a static method if (klassProto.hasOwnProperty("__new__")) { const newf = klassProto.__new__; @@ -221,6 +231,9 @@ function tp$new(args, kwargs) { } klass.$allocateSlots(); + set_names(klass); + init_subclass(klass, kwargs); + return klass; } @@ -587,7 +600,11 @@ function $allocateSlot(dunder, dunderFunc) { if (proto.hasOwnProperty(slot_name)) { delete proto[slot_name]; // required in order to override the multiple inheritance getter slots } - proto[slot_name] = slot_def.$slot_func(dunderFunc); + Object.defineProperty(proto, slot_name, { + value: slot_def.$slot_func(dunderFunc), + writable: true, + configurable: true, + }); } function $allocateGetterSlot(dunder) { @@ -761,6 +778,15 @@ Sk.builtin.type.prototype.tp$methods = /**@lends {Sk.builtin.type.prototype}*/ { }, }; +Sk.builtin.type.tp$classmethods = { + __prepare__: { + $meth() { + return new Sk.builtin.dict([]); + }, + $flags: { FastCall: true }, + }, +}; + // similar to generic.getSetDict but have to check if there is a builtin __dict__ descriptor that we should use first! const subtype_dict_getset_description = { $get() { @@ -805,3 +831,27 @@ function check_special_type_attr(type, value, pyName) { throw new Sk.builtin.TypeError("can't delete " + type.prototype.tp$name + "." + pyName.$jsstr()); } } + +function init_subclass(type, kws) { + const super_ = new Sk.builtin.super_(type, type); + const func = super_.tp$getattr(Sk.builtin.str.$initsubclass); + Sk.misceval.callsimArray(func, [], kws); +} + +function set_names(type) { + const proto = type.prototype; + Object.keys(proto).forEach((key) => { + const set_func = Sk.abstr.lookupSpecial(proto[key], Sk.builtin.str.$setname); + if (set_func !== undefined) { + try { + Sk.misceval.callsimArray(set_func, [type, new Sk.builtin.str(key)]); + } catch (e) { + const runtime_err = new Sk.builtin.RuntimeError( + "Error calling __set_name__ on '" + Sk.abstr.typeName(proto[key]) + "' instance '" + key + "' in '" + type.prototype.tp$name + "'" + ); + runtime_err.$cause = e; + throw runtime_err; + } + } + }); +} \ No newline at end of file diff --git a/test/run/t407.py.real b/test/run/t407.py.real index e97614f7a5..72a366b017 100644 --- a/test/run/t407.py.real +++ b/test/run/t407.py.real @@ -1,3 +1,3 @@ -['__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__gt__', '__hash__', '__init__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__repr__', '__setattr__', '__str__', 'a', 'b', 'c'] -['__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__gt__', '__hash__', '__init__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__repr__', '__setattr__', '__str__', 'a', 'b', 'c', 'd'] +['__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__repr__', '__setattr__', '__str__', 'a', 'b', 'c'] +['__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__repr__', '__setattr__', '__str__', 'a', 'b', 'c', 'd'] ['a', 'b', 'c', 'd'] diff --git a/test/run/t407.py.real.force b/test/run/t407.py.real.force index e97614f7a5..72a366b017 100644 --- a/test/run/t407.py.real.force +++ b/test/run/t407.py.real.force @@ -1,3 +1,3 @@ -['__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__gt__', '__hash__', '__init__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__repr__', '__setattr__', '__str__', 'a', 'b', 'c'] -['__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__gt__', '__hash__', '__init__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__repr__', '__setattr__', '__str__', 'a', 'b', 'c', 'd'] +['__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__repr__', '__setattr__', '__str__', 'a', 'b', 'c'] +['__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__repr__', '__setattr__', '__str__', 'a', 'b', 'c', 'd'] ['a', 'b', 'c', 'd'] diff --git a/test/unit3/test_descr.py b/test/unit3/test_descr.py index 9b05550dc3..0513e6633d 100644 --- a/test/unit3/test_descr.py +++ b/test/unit3/test_descr.py @@ -18,107 +18,6 @@ # except ImportError: # _testcapi = None -# moved to global scope when #1178 is fixed this can be moved back into test_doc_descriptor -class DocDescr(object): - def __get__(self, object, otype): - if object: - object = object.__class__.__name__ + ' instance' - if otype: - otype = otype.__name__ - return 'object=%s; type=%s' % (object, otype) - -class Descr(object): - - def __init__(self, name): - self.name = name - - def __set__(self, obj, value): - obj.__dict__[self.name] = value -descr = Descr("a") - - -class GetattrDescriptor(object): - counter = 0 - def __get__(self, obj, objtype=None): - def getter(name): - self.counter += 1 - raise AttributeError(name) - return getter - -getattr_descr = GetattrDescriptor() - -class C_methods(object): - def __init__(self, x): - self.x = x - def foo(self): - return self.x -c1_methods = C_methods(1) - - -_ok = None -class Checker(object): - def __init__(self, test, ok=set()): - self.test = test - global _ok - _ok = ok - def __getattr__(self, attr): - self.test.fail("__getattr__ called with {0}".format(attr)) - def __getattribute__(self, attr): - if attr not in _ok: - self.test.fail("__getattribute__ called with {0}".format(attr)) - return object.__getattribute__(self, attr) - -class SpecialDescr(object): - def __init__(self, impl, record): - self.impl = impl - self.record = record - def __get__(self, obj, owner): - self.record.append(1) - return self.impl.__get__(obj, owner) -class MyException(Exception): - pass -class ErrDescr(object): - def __get__(self, obj, owner): - raise MyException - - -#1178 from property plus -class C(object): - foo = property(doc="hello") - @foo.getter - def foo(self): - return self._foo - @foo.setter - def foo(self, value): - self._foo = abs(value) - @foo.deleter - def foo(self): - del self._foo - - -class E(object): - @property - def foo(self): - return self._foo - @foo.setter - def foo(self, value): - raise RuntimeError - @foo.setter - def foo(self, value): - self._foo = abs(value) - @foo.deleter - def foo(self, value=None): - del self._foo - - -class C_Multi(object): - def __init__(self): - self.__state = 0 - def getstate(self): - return self.__state - def setstate(self, state): - self.__state = state - class OperatorsTest(unittest.TestCase): def __init__(self, *args, **kwargs): @@ -577,347 +476,349 @@ def __getitem__(self, i): self.assertEqual(a[2], 102) self.assertEqual(a[100:200], (100,200)) - # def test_metaclass(self): - # # Testing metaclasses... - # class C(metaclass=type): - # def __init__(self): - # self.__state = 0 - # def getstate(self): - # return self.__state - # def setstate(self, state): - # self.__state = state - # a = C() - # self.assertEqual(a.getstate(), 0) - # a.setstate(10) - # self.assertEqual(a.getstate(), 10) - # class _metaclass(type): - # def myself(cls): return cls - # class D(metaclass=_metaclass): - # pass - # self.assertEqual(D.myself(), D) - # d = D() - # self.assertEqual(d.__class__, D) - # class M1(type): - # def __new__(cls, name, bases, dict): - # dict['__spam__'] = 1 - # return type.__new__(cls, name, bases, dict) - # class C(metaclass=M1): - # pass - # self.assertEqual(C.__spam__, 1) - # c = C() - # self.assertEqual(c.__spam__, 1) + def test_metaclass(self): + # Testing metaclasses... + global A, B, C, D, E, M1, M2, T, autosuper, autoproperty, multimetaclass, _instance, AMeta, BMeta, C2 #1178 + global ANotMeta, BNotMeta, func + class C(metaclass=type): + def __init__(self): + self.__state = 0 + def getstate(self): + return self.__state + def setstate(self, state): + self.__state = state + a = C() + self.assertEqual(a.getstate(), 0) + a.setstate(10) + self.assertEqual(a.getstate(), 10) + class _metaclass(type): + def myself(cls): return cls + class D(metaclass=_metaclass): + pass + self.assertEqual(D.myself(), D) + d = D() + self.assertEqual(d.__class__, D) + class M1(type): + def __new__(cls, name, bases, dict): + dict['__spam__'] = 1 + return type.__new__(cls, name, bases, dict) + class C(metaclass=M1): + pass + self.assertEqual(C.__spam__, 1) + c = C() + self.assertEqual(c.__spam__, 1) - # class _instance(object): - # pass - # class M2(object): - # @staticmethod - # def __new__(cls, name, bases, dict): - # self = object.__new__(cls) - # self.name = name - # self.bases = bases - # self.dict = dict - # return self - # def __call__(self): - # it = _instance() - # # Early binding of methods - # for key in self.dict: - # if key.startswith("__"): - # continue - # setattr(it, key, self.dict[key].__get__(it, self)) - # return it - # class C(metaclass=M2): - # def spam(self): - # return 42 - # self.assertEqual(C.name, 'C') - # self.assertEqual(C.bases, ()) - # self.assertIn('spam', C.dict) - # c = C() - # self.assertEqual(c.spam(), 42) - - # # More metaclass examples - - # class autosuper(type): - # # Automatically add __super to the class - # # This trick only works for dynamic classes - # def __new__(metaclass, name, bases, dict): - # cls = super(autosuper, metaclass).__new__(metaclass, - # name, bases, dict) - # # Name mangling for __super removes leading underscores - # while name[:1] == "_": - # name = name[1:] - # if name: - # name = "_%s__super" % name - # else: - # name = "__super" - # setattr(cls, name, super(cls)) - # return cls - # class A(metaclass=autosuper): - # def meth(self): - # return "A" - # class B(A): - # def meth(self): - # return "B" + self.__super.meth() - # class C(A): - # def meth(self): - # return "C" + self.__super.meth() - # class D(C, B): - # def meth(self): - # return "D" + self.__super.meth() - # self.assertEqual(D().meth(), "DCBA") - # class E(B, C): - # def meth(self): - # return "E" + self.__super.meth() - # self.assertEqual(E().meth(), "EBCA") - - # class autoproperty(type): - # # Automatically create property attributes when methods - # # named _get_x and/or _set_x are found - # def __new__(metaclass, name, bases, dict): - # hits = {} - # for key, val in dict.items(): - # if key.startswith("_get_"): - # key = key[5:] - # get, set = hits.get(key, (None, None)) - # get = val - # hits[key] = get, set - # elif key.startswith("_set_"): - # key = key[5:] - # get, set = hits.get(key, (None, None)) - # set = val - # hits[key] = get, set - # for key, (get, set) in hits.items(): - # dict[key] = property(get, set) - # return super(autoproperty, metaclass).__new__(metaclass, - # name, bases, dict) - # class A(metaclass=autoproperty): - # def _get_x(self): - # return -self.__x - # def _set_x(self, x): - # self.__x = -x - # a = A() - # self.assertNotHasAttr(a, "x") - # a.x = 12 - # self.assertEqual(a.x, 12) - # self.assertEqual(a._A__x, -12) - - # class multimetaclass(autoproperty, autosuper): - # # Merge of multiple cooperating metaclasses - # pass - # class A(metaclass=multimetaclass): - # def _get_x(self): - # return "A" - # class B(A): - # def _get_x(self): - # return "B" + self.__super._get_x() - # class C(A): - # def _get_x(self): - # return "C" + self.__super._get_x() - # class D(C, B): - # def _get_x(self): - # return "D" + self.__super._get_x() - # self.assertEqual(D().x, "DCBA") - - # # Make sure type(x) doesn't call x.__class__.__init__ - # class T(type): - # counter = 0 - # def __init__(self, *args): - # T.counter += 1 - # class C(metaclass=T): - # pass - # self.assertEqual(T.counter, 1) - # a = C() - # self.assertEqual(type(a), C) - # self.assertEqual(T.counter, 1) + class _instance(object): + pass + class M2(object): + @staticmethod + def __new__(cls, name, bases, dict): + self = object.__new__(cls) + self.name = name + self.bases = bases + self.dict = dict + return self + def __call__(self): + it = _instance() + # Early binding of methods + for key in self.dict: + if key.startswith("__"): + continue + setattr(it, key, self.dict[key].__get__(it, self)) + return it + class C(metaclass=M2): + def spam(self): + return 42 + self.assertEqual(C.name, 'C') + self.assertEqual(C.bases, ()) + self.assertIn('spam', C.dict) + c = C() + self.assertEqual(c.spam(), 42) + + # More metaclass examples + + class autosuper(type): + # Automatically add __super to the class + # This trick only works for dynamic classes + def __new__(metaclass, name, bases, dict): + cls = super(autosuper, metaclass).__new__(metaclass, + name, bases, dict) + # Name mangling for __super removes leading underscores + while name[:1] == "_": + name = name[1:] + if name: + name = "_%s__super" % name + else: + name = "__super" + setattr(cls, name, super(cls)) + return cls + class A(metaclass=autosuper): + def meth(self): + return "A" + class B(A): + def meth(self): + return "B" + self.__super.meth() + class C(A): + def meth(self): + return "C" + self.__super.meth() + class D(C, B): + def meth(self): + return "D" + self.__super.meth() + self.assertEqual(D().meth(), "DCBA") + class E(B, C): + def meth(self): + return "E" + self.__super.meth() + self.assertEqual(E().meth(), "EBCA") + + class autoproperty(type): + # Automatically create property attributes when methods + # named _get_x and/or _set_x are found + def __new__(metaclass, name, bases, dict): + hits = {} + for key, val in dict.items(): + if key.startswith("_get_"): + key = key[5:] + get, set = hits.get(key, (None, None)) + get = val + hits[key] = get, set + elif key.startswith("_set_"): + key = key[5:] + get, set = hits.get(key, (None, None)) + set = val + hits[key] = get, set + for key, (get, set) in hits.items(): + dict[key] = property(get, set) + return super(autoproperty, metaclass).__new__(metaclass, + name, bases, dict) + class A(metaclass=autoproperty): + def _get_x(self): + return -self.__x + def _set_x(self, x): + self.__x = -x + a = A() + self.assertNotHasAttr(a, "x") + a.x = 12 + self.assertEqual(a.x, 12) + self.assertEqual(a._A__x, -12) - # class C(object): pass - # c = C() - # try: c() - # except TypeError: pass - # else: self.fail("calling object w/o call method should raise " - # "TypeError") + class multimetaclass(autoproperty, autosuper): + # Merge of multiple cooperating metaclasses + pass + class A(metaclass=multimetaclass): + def _get_x(self): + return "A" + class B(A): + def _get_x(self): + return "B" + self.__super._get_x() + class C(A): + def _get_x(self): + return "C" + self.__super._get_x() + class D(C, B): + def _get_x(self): + return "D" + self.__super._get_x() + self.assertEqual(D().x, "DCBA") - # # Testing code to find most derived baseclass - # class A(type): - # def __new__(*args, **kwargs): - # return type.__new__(*args, **kwargs) + # Make sure type(x) doesn't call x.__class__.__init__ + class T(type): + counter = 0 + def __init__(self, *args): + T.counter += 1 + class C(metaclass=T): + pass + self.assertEqual(T.counter, 1) + a = C() + self.assertEqual(type(a), C) + self.assertEqual(T.counter, 1) - # class B(object): - # pass + class C(object): pass + c = C() + try: c() + except TypeError: pass + else: self.fail("calling object w/o call method should raise " + "TypeError") - # class C(object, metaclass=A): - # pass + # Testing code to find most derived baseclass + class A(type): + def __new__(*args, **kwargs): + return type.__new__(*args, **kwargs) - # # The most derived metaclass of D is A rather than type. - # class D(B, C): - # pass - # self.assertIs(A, type(D)) - - # # issue1294232: correct metaclass calculation - # new_calls = [] # to check the order of __new__ calls - # class AMeta(type): - # @staticmethod - # def __new__(mcls, name, bases, ns): - # new_calls.append('AMeta') - # return super().__new__(mcls, name, bases, ns) - # @classmethod - # def __prepare__(mcls, name, bases): - # return {} - - # class BMeta(AMeta): - # @staticmethod - # def __new__(mcls, name, bases, ns): - # new_calls.append('BMeta') - # return super().__new__(mcls, name, bases, ns) - # @classmethod - # def __prepare__(mcls, name, bases): - # ns = super().__prepare__(name, bases) - # ns['BMeta_was_here'] = True - # return ns - - # class A(metaclass=AMeta): - # pass - # self.assertEqual(['AMeta'], new_calls) - # new_calls.clear() + class B(object): + pass - # class B(metaclass=BMeta): - # pass - # # BMeta.__new__ calls AMeta.__new__ with super: - # self.assertEqual(['BMeta', 'AMeta'], new_calls) - # new_calls.clear() + class C(object, metaclass=A): + pass - # class C(A, B): - # pass - # # The most derived metaclass is BMeta: - # self.assertEqual(['BMeta', 'AMeta'], new_calls) - # new_calls.clear() - # # BMeta.__prepare__ should've been called: - # self.assertIn('BMeta_was_here', C.__dict__) - - # # The order of the bases shouldn't matter: - # class C2(B, A): - # pass - # self.assertEqual(['BMeta', 'AMeta'], new_calls) - # new_calls.clear() - # self.assertIn('BMeta_was_here', C2.__dict__) + # The most derived metaclass of D is A rather than type. + class D(B, C): + pass + self.assertIs(A, type(D)) - # # Check correct metaclass calculation when a metaclass is declared: - # class D(C, metaclass=type): - # pass - # self.assertEqual(['BMeta', 'AMeta'], new_calls) - # new_calls.clear() - # self.assertIn('BMeta_was_here', D.__dict__) + # issue1294232: correct metaclass calculation + new_calls = [] # to check the order of __new__ calls + class AMeta(type): + @staticmethod + def __new__(mcls, name, bases, ns): + new_calls.append('AMeta') + return super().__new__(mcls, name, bases, ns) + @classmethod + def __prepare__(mcls, name, bases): + return {} - # class E(C, metaclass=AMeta): - # pass - # self.assertEqual(['BMeta', 'AMeta'], new_calls) - # new_calls.clear() - # self.assertIn('BMeta_was_here', E.__dict__) - - # # Special case: the given metaclass isn't a class, - # # so there is no metaclass calculation. - # marker = object() - # def func(*args, **kwargs): - # return marker - # class X(metaclass=func): - # pass - # class Y(object, metaclass=func): - # pass - # class Z(D, metaclass=func): - # pass - # self.assertIs(marker, X) - # self.assertIs(marker, Y) - # self.assertIs(marker, Z) - - # # The given metaclass is a class, - # # but not a descendant of type. - # prepare_calls = [] # to track __prepare__ calls - # class ANotMeta: - # def __new__(mcls, *args, **kwargs): - # new_calls.append('ANotMeta') - # return super().__new__(mcls) - # @classmethod - # def __prepare__(mcls, name, bases): - # prepare_calls.append('ANotMeta') - # return {} - # class BNotMeta(ANotMeta): - # def __new__(mcls, *args, **kwargs): - # new_calls.append('BNotMeta') - # return super().__new__(mcls) - # @classmethod - # def __prepare__(mcls, name, bases): - # prepare_calls.append('BNotMeta') - # return super().__prepare__(name, bases) - - # class A(metaclass=ANotMeta): - # pass - # self.assertIs(ANotMeta, type(A)) - # self.assertEqual(['ANotMeta'], prepare_calls) - # prepare_calls.clear() - # self.assertEqual(['ANotMeta'], new_calls) - # new_calls.clear() + class BMeta(AMeta): + @staticmethod + def __new__(mcls, name, bases, ns): + new_calls.append('BMeta') + return super().__new__(mcls, name, bases, ns) + @classmethod + def __prepare__(mcls, name, bases): + ns = super().__prepare__(name, bases) + ns['BMeta_was_here'] = True + return ns - # class B(metaclass=BNotMeta): - # pass - # self.assertIs(BNotMeta, type(B)) - # self.assertEqual(['BNotMeta', 'ANotMeta'], prepare_calls) - # prepare_calls.clear() - # self.assertEqual(['BNotMeta', 'ANotMeta'], new_calls) - # new_calls.clear() + class A(metaclass=AMeta): + pass + self.assertEqual(['AMeta'], new_calls) + new_calls.clear() - # class C(A, B): - # pass - # self.assertIs(BNotMeta, type(C)) - # self.assertEqual(['BNotMeta', 'ANotMeta'], new_calls) - # new_calls.clear() - # self.assertEqual(['BNotMeta', 'ANotMeta'], prepare_calls) - # prepare_calls.clear() + class B(metaclass=BMeta): + pass + # BMeta.__new__ calls AMeta.__new__ with super: + self.assertEqual(['BMeta', 'AMeta'], new_calls) + new_calls.clear() - # class C2(B, A): - # pass - # self.assertIs(BNotMeta, type(C2)) - # self.assertEqual(['BNotMeta', 'ANotMeta'], new_calls) - # new_calls.clear() - # self.assertEqual(['BNotMeta', 'ANotMeta'], prepare_calls) - # prepare_calls.clear() - - # # This is a TypeError, because of a metaclass conflict: - # # BNotMeta is neither a subclass, nor a superclass of type - # with self.assertRaises(TypeError): - # class D(C, metaclass=type): - # pass + class C(A, B): + pass + # The most derived metaclass is BMeta: + self.assertEqual(['BMeta', 'AMeta'], new_calls) + new_calls.clear() + # BMeta.__prepare__ should've been called: + self.assertIn('BMeta_was_here', C.__dict__) + + # The order of the bases shouldn't matter: + class C2(B, A): + pass + self.assertEqual(['BMeta', 'AMeta'], new_calls) + new_calls.clear() + self.assertIn('BMeta_was_here', C2.__dict__) - # class E(C, metaclass=ANotMeta): - # pass - # self.assertIs(BNotMeta, type(E)) - # self.assertEqual(['BNotMeta', 'ANotMeta'], new_calls) - # new_calls.clear() - # self.assertEqual(['BNotMeta', 'ANotMeta'], prepare_calls) - # prepare_calls.clear() + # Check correct metaclass calculation when a metaclass is declared: + class D(C, metaclass=type): + pass + self.assertEqual(['BMeta', 'AMeta'], new_calls) + new_calls.clear() + self.assertIn('BMeta_was_here', D.__dict__) - # class F(object(), C): - # pass - # self.assertIs(BNotMeta, type(F)) - # self.assertEqual(['BNotMeta', 'ANotMeta'], new_calls) - # new_calls.clear() - # self.assertEqual(['BNotMeta', 'ANotMeta'], prepare_calls) - # prepare_calls.clear() + class E(C, metaclass=AMeta): + pass + self.assertEqual(['BMeta', 'AMeta'], new_calls) + new_calls.clear() + self.assertIn('BMeta_was_here', E.__dict__) + + # Special case: the given metaclass isn't a class, + # so there is no metaclass calculation. + marker = object() + def func(*args, **kwargs): + return marker + class X(metaclass=func): + pass + class Y(object, metaclass=func): + pass + class Z(D, metaclass=func): + pass + self.assertIs(marker, X) + self.assertIs(marker, Y) + self.assertIs(marker, Z) + + # The given metaclass is a class, + # but not a descendant of type. + prepare_calls = [] # to track __prepare__ calls + class ANotMeta: + def __new__(mcls, *args, **kwargs): + new_calls.append('ANotMeta') + return super().__new__(mcls) + @classmethod + def __prepare__(mcls, name, bases): + prepare_calls.append('ANotMeta') + return {} + class BNotMeta(ANotMeta): + def __new__(mcls, *args, **kwargs): + new_calls.append('BNotMeta') + return super().__new__(mcls) + @classmethod + def __prepare__(mcls, name, bases): + prepare_calls.append('BNotMeta') + return super().__prepare__(name, bases) - # class F2(C, object()): - # pass - # self.assertIs(BNotMeta, type(F2)) - # self.assertEqual(['BNotMeta', 'ANotMeta'], new_calls) - # new_calls.clear() - # self.assertEqual(['BNotMeta', 'ANotMeta'], prepare_calls) - # prepare_calls.clear() - - # # TypeError: BNotMeta is neither a - # # subclass, nor a superclass of int - # with self.assertRaises(TypeError): - # class X(C, int()): - # pass - # with self.assertRaises(TypeError): - # class X(int(), C): - # pass + class A(metaclass=ANotMeta): + pass + self.assertIs(ANotMeta, type(A)) + self.assertEqual(['ANotMeta'], prepare_calls) + prepare_calls.clear() + self.assertEqual(['ANotMeta'], new_calls) + new_calls.clear() + + class B(metaclass=BNotMeta): + pass + self.assertIs(BNotMeta, type(B)) + self.assertEqual(['BNotMeta', 'ANotMeta'], prepare_calls) + prepare_calls.clear() + self.assertEqual(['BNotMeta', 'ANotMeta'], new_calls) + new_calls.clear() + + class C(A, B): + pass + self.assertIs(BNotMeta, type(C)) + self.assertEqual(['BNotMeta', 'ANotMeta'], new_calls) + new_calls.clear() + self.assertEqual(['BNotMeta', 'ANotMeta'], prepare_calls) + prepare_calls.clear() + + class C2(B, A): + pass + self.assertIs(BNotMeta, type(C2)) + self.assertEqual(['BNotMeta', 'ANotMeta'], new_calls) + new_calls.clear() + self.assertEqual(['BNotMeta', 'ANotMeta'], prepare_calls) + prepare_calls.clear() + + # This is a TypeError, because of a metaclass conflict: + # BNotMeta is neither a subclass, nor a superclass of type + with self.assertRaises(TypeError): + class D(C, metaclass=type): + pass + + class E(C, metaclass=ANotMeta): + pass + self.assertIs(BNotMeta, type(E)) + self.assertEqual(['BNotMeta', 'ANotMeta'], new_calls) + new_calls.clear() + self.assertEqual(['BNotMeta', 'ANotMeta'], prepare_calls) + prepare_calls.clear() + + class F(object(), C): + pass + self.assertIs(BNotMeta, type(F)) + self.assertEqual(['BNotMeta', 'ANotMeta'], new_calls) + new_calls.clear() + self.assertEqual(['BNotMeta', 'ANotMeta'], prepare_calls) + prepare_calls.clear() + + class F2(C, object()): + pass + self.assertIs(BNotMeta, type(F2)) + self.assertEqual(['BNotMeta', 'ANotMeta'], new_calls) + new_calls.clear() + self.assertEqual(['BNotMeta', 'ANotMeta'], prepare_calls) + prepare_calls.clear() + + # TypeError: BNotMeta is neither a + # subclass, nor a superclass of int + with self.assertRaises(TypeError): + class X(C, int()): + pass + with self.assertRaises(TypeError): + class X(int(), C): + pass def test_module_subclasses(self): # Testing Python subclass of module... @@ -955,14 +856,14 @@ class Module(types.ModuleType, str): def test_multiple_inheritance(self): # Testing multiple inheritance... - # moved to global scope #1178 - # class C_Multi(object): - # def __init__(self): - # self.__state = 0 - # def getstate(self): - # return self.__state - # def setstate(self, state): - # self.__state = state + global C_Multi #1178 + class C_Multi(object): + def __init__(self): + self.__state = 0 + def getstate(self): + return self.__state + def setstate(self, state): + self.__state = state a = C_Multi() self.assertEqual(a.getstate(), 0) a.setstate(10) @@ -1203,6 +1104,7 @@ class MyFrozenSet(frozenset): def test_slots(self): # Testing __slots__... + global C, slots #1178 class C0(object): __slots__ = [] x = C0() @@ -1405,6 +1307,7 @@ class C(object): def test_slots_special(self): # Testing __dict__ and __weakref__ in __slots__... + global D, C, C2 class D(object): __slots__ = ["__dict__"] a = D() @@ -1562,11 +1465,12 @@ class I(int): self.assertEqual(I(3)*I(2), 6) # Test comparison of classes with dynamic metaclasses - # class dynamicmetaclass(type): - # pass - # class someclass(metaclass=dynamicmetaclass): - # pass - # self.assertNotEqual(someclass, object) + global dynamicmetaclass #1178 + class dynamicmetaclass(type): + pass + class someclass(metaclass=dynamicmetaclass): + pass + self.assertNotEqual(someclass, object) def test_errors(self): # Testing errors... @@ -1612,21 +1516,22 @@ class C(object): else: self.fail("__slots__ = [1] should be illegal") - # class M1(type): - # pass - # class M2(type): - # pass - # class A1(object, metaclass=M1): - # pass - # class A2(object, metaclass=M2): - # pass - # try: - # class B(A1, A2): - # pass - # except TypeError: - # pass - # else: - # self.fail("finding the most derived metaclass should have failed") + global M1, M2, A1, A2, B #1178 + class M1(type): + pass + class M2(type): + pass + class A1(object, metaclass=M1): + pass + class A2(object, metaclass=M2): + pass + try: + class B(A1, A2): + pass + except TypeError: + pass + else: + self.fail("finding the most derived metaclass should have failed") def test_classmethods(self): # Testing class methods... @@ -2003,6 +1908,7 @@ class D(B, C): def test_overloading(self): # Testing operator overloading... + global B, C class B(object): "Intermediate class because object doesn't have a __setattr__" @@ -2052,13 +1958,13 @@ def __delitem__(self, key): def test_methods(self): # Testing methods... - # Skulpt #1178 moving this to global scope - # class C_methods(object): - # def __init__(self, x): - # self.x = x - # def foo(self): - # return self.x - # c1_methods = C_methods(1) + global C_methods, c1_methods #1178 moving this to global scope + class C_methods(object): + def __init__(self, x): + self.x = x + def foo(self): + return self.x + c1_methods = C_methods(1) self.assertEqual(c1_methods.foo(), 1) class D(C_methods): boo = C_methods.foo @@ -2159,24 +2065,32 @@ def format_impl(self, spec): ] # Skulpt #1178 moved to global scope - # class Checker(object): - # def __getattr__(self, attr, test=self): - # test.fail("__getattr__ called with {0}".format(attr)) - # def __getattribute__(self, attr, test=self): - # if attr not in ok: - # test.fail("__getattribute__ called with {0}".format(attr)) - # return object.__getattribute__(self, attr) - # class SpecialDescr(object): - # def __init__(self, impl): - # self.impl = impl - # def __get__(self, obj, owner): - # record.append(1) - # return self.impl.__get__(obj, owner) - # class MyException(Exception): - # pass - # class ErrDescr(object): - # def __get__(self, obj, owner): - # raise MyException + global Checker, SpecialDescr, MyException, ErrDescr, X + global _self + _self = self + class Checker(object): + def __init__(self, test, ok=set()): + self.test = test + global _ok + _ok = ok + def __getattr__(self, attr, test=_self): + test.fail("__getattr__ called with {0}".format(attr)) + def __getattribute__(self, attr, test=_self): + if attr not in ok: + test.fail("__getattribute__ called with {0}".format(attr)) + return object.__getattribute__(self, attr) + class SpecialDescr(object): + def __init__(self, impl, record): + self.impl = impl + def __get__(self, obj, owner): + record.append(1) + return self.impl.__get__(obj, owner) + class MyException(Exception): + pass + class ErrDescr(object): + def __get__(self, obj, owner): + + raise MyException for name, runner, meth_impl, ok, env in specials: class X(Checker): @@ -2421,6 +2335,7 @@ class D(object): # p = property(_testcapi.test_with_docstring) def test_properties_plus(self): + global C, D, E, F class C(object): foo = property(doc="hello") @foo.getter @@ -2669,6 +2584,7 @@ def getdict(self): def test_supers(self): # Testing super... + global A, B, C, D, E, F # 1178 class A(object): def meth(self, a): @@ -3333,13 +3249,14 @@ def test_doc_descriptor(self): # Testing __doc__ descriptor... # SF bug 542984 # Skulpt BUG #1178 move this to global scope - # class DocDescr(object): - # def __get__(self, object, otype): - # if object: - # object = object.__class__.__name__ + ' instance' - # if otype: - # otype = otype.__name__ - # return 'object=%s; type=%s' % (object, otype) + global DocDescr + class DocDescr(object): + def __get__(self, object, otype): + if object: + object = object.__class__.__name__ + ' instance' + if otype: + otype = otype.__name__ + return 'object=%s; type=%s' % (object, otype) class OldClass: __doc__ = DocDescr() class NewClass(object): @@ -3446,6 +3363,7 @@ class R(J): def test_set_dict(self): # Testing __dict__ assignment... + global Base, Meta1, Meta2, D, E, C, Module1, Module2, Exception1, Exception2 #1178 class C(object): pass a = C() a.__dict__ = {'b': 1} @@ -3464,6 +3382,7 @@ def cant(x, dict): class Base(object): pass + def verify_dict_readonly(x): """ x has to be an instance of a class inheriting from Base. @@ -3483,24 +3402,25 @@ def verify_dict_readonly(x): else: self.fail("dict_descr allowed access to %r's dict" % x) - # # Classes don't allow __dict__ assignment and have readonly dicts - # class Meta1(type, Base): - # pass - # class Meta2(Base, type): - # pass - # class D(object, metaclass=Meta1): - # pass - # class E(object, metaclass=Meta2): - # pass - # for cls in C, D, E: - # verify_dict_readonly(cls) - # class_dict = cls.__dict__ - # try: - # class_dict["spam"] = "eggs" - # except TypeError: - # pass - # else: - # self.fail("%r's __dict__ can be modified" % cls) + # Classes don't allow __dict__ assignment and have readonly dicts + + class Meta1(type, Base): + pass + class Meta2(Base, type): + pass + class D(object, metaclass=Meta1): + pass + class E(object, metaclass=Meta2): + pass + for cls in C, D, E: + verify_dict_readonly(cls) + class_dict = cls.__dict__ + try: + class_dict["spam"] = "eggs" + except TypeError: + pass + else: + self.fail("%r's __dict__ can be modified" % cls) # Modules also disallow __dict__ assignment class Module1(types.ModuleType, Base): @@ -4710,15 +4630,16 @@ def __setitem__(self, idx, value): def test_set_and_no_get(self): # See # http://mail.python.org/pipermail/python-dev/2010-January/095637.html + global Descr, X, Meta, descr #moved to global scope see skulpt #1178 - # class Descr(object): + class Descr(object): - # def __init__(self, name): - # self.name = name + def __init__(self, name): + self.name = name - # def __set__(self, obj, value): - # obj.__dict__[self.name] = value - # descr = Descr("a") + def __set__(self, obj, value): + obj.__dict__[self.name] = value + descr = Descr("a") class X(object): a = descr @@ -4729,27 +4650,28 @@ class X(object): self.assertEqual(x.a, 42) # Also check type_getattro for correctness. - # class Meta(type): - # pass - # class X(metaclass=Meta): - # pass - # X.a = 42 - # Meta.a = Descr("a") - # self.assertEqual(X.a, 42) + class Meta(type): + pass + class X(metaclass=Meta): + pass + X.a = 42 + Meta.a = Descr("a") + self.assertEqual(X.a, 42) def test_getattr_hooks(self): # issue 4230 # moved to global scope # skulpt #1178 - # class GetattrDescriptor(object): - # counter = 0 - # def __get__(self, obj, objtype=None): - # def getter(name): - # self.counter += 1 - # raise AttributeError(name) - # return getter - - # getattr_descr = GetattrDescriptor() + global getattr_descr, GetattrDescriptor + class GetattrDescriptor(object): + counter = 0 + def __get__(self, obj, objtype=None): + def getter(name): + self.counter += 1 + raise AttributeError(name) + return getter + + getattr_descr = GetattrDescriptor() class A(object): __getattribute__ = getattr_descr class B(object): @@ -5056,16 +4978,17 @@ def meth(self): # self.assertEqual(keys, ['__dict__', '__doc__', '__module__', # '__weakref__', 'meth']) - # def test_dict_type_with_metaclass(self): - # # Testing type of __dict__ when metaclass set... - # class B(object): - # pass - # class M(type): - # pass - # class C(metaclass=M): - # # In 2.3a1, C.__dict__ was a real dict rather than a dict proxy - # pass - # self.assertEqual(type(C.__dict__), type(B.__dict__)) + def test_dict_type_with_metaclass(self): + # Testing type of __dict__ when metaclass set... + global M + class B(object): + pass + class M(type): + pass + class C(metaclass=M): + # In 2.3a1, C.__dict__ was a real dict rather than a dict proxy + pass + self.assertEqual(type(C.__dict__), type(B.__dict__)) def test_repr(self): # Testing mappingproxy.__repr__. diff --git a/test/unit3/test_init_subclass.py b/test/unit3/test_init_subclass.py new file mode 100644 index 0000000000..be465ca4db --- /dev/null +++ b/test/unit3/test_init_subclass.py @@ -0,0 +1,301 @@ +import types +import unittest + + +# argh bug #1178 is very annoying for these tests!! +# when #1178 is fixed can remove all the global + + +class Test(unittest.TestCase): + def test_init_subclass(self): + global A,B + class A: + initialized = False + + def __init_subclass__(cls): + super().__init_subclass__() + cls.initialized = True + + class B(A): + pass + self.assertFalse(A.initialized) + self.assertTrue(B.initialized) + + def test_init_subclass_dict(self): + global A,B + class A(dict): + initialized = False + + def __init_subclass__(cls): + super().__init_subclass__() + cls.initialized = True + + class B(A): + pass + self.assertFalse(A.initialized) + self.assertTrue(B.initialized) + + def test_init_subclass_kwargs(self): + global A,B + class A: + def __init_subclass__(cls, **kwargs): + cls.kwargs = kwargs + + class B(A, x=3): + pass + + self.assertEqual(B.kwargs, dict(x=3)) + + def test_init_subclass_error(self): + global A,B + class A: + def __init_subclass__(cls): + raise RuntimeError + + with self.assertRaises(RuntimeError): + class B(A): + pass + + def test_init_subclass_wrong(self): + global A,B + class A: + def __init_subclass__(cls, whatever): + pass + + with self.assertRaises(TypeError): + class B(A): + pass + + def test_init_subclass_skipped(self): + global BaseWithInit, BaseWithoutInit, A + class BaseWithInit: + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + cls.initialized = cls + + class BaseWithoutInit(BaseWithInit): + pass + + class A(BaseWithoutInit): + pass + + self.assertIs(A.initialized, A) + self.assertIs(BaseWithoutInit.initialized, BaseWithoutInit) + + def test_init_subclass_diamond(self): + global Base, Left, Middle, Right, A + class Base: + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + cls.calls = [] + + class Left(Base): + pass + + class Middle: + def __init_subclass__(cls, middle, **kwargs): + super().__init_subclass__(**kwargs) + cls.calls += [middle] + + class Right(Base): + def __init_subclass__(cls, right="right", **kwargs): + super().__init_subclass__(**kwargs) + cls.calls += [right] + + class A(Left, Middle, Right, middle="middle"): + pass + + self.assertEqual(A.calls, ["right", "middle"]) + self.assertEqual(Left.calls, []) + self.assertEqual(Right.calls, []) + + def test_set_name(self): + global Descriptor + class Descriptor: + def __set_name__(self, owner, name): + self.owner = owner + self.name = name + + class A: + d = Descriptor() + + self.assertEqual(A.d.name, "d") + self.assertIs(A.d.owner, A) + + def test_set_name_metaclass(self): + global Meta, Descriptor + class Meta(type): + def __new__(cls, name, bases, ns): + ret = super().__new__(cls, name, bases, ns) + self.assertEqual(ret.d.name, "d") + self.assertIs(ret.d.owner, ret) + return 0 + + class Descriptor: + def __set_name__(self, owner, name): + self.owner = owner + self.name = name + + class A(metaclass=Meta): + d = Descriptor() + self.assertEqual(A, 0) + + def test_set_name_error(self): + global Descriptor, NotGoingToWork + class Descriptor: + def __set_name__(self, owner, name): + 1/0 + + with self.assertRaises(RuntimeError) as cm: + class NotGoingToWork: + attr = Descriptor() + + exc = cm.exception + self.assertRegex(str(exc), r'\bNotGoingToWork\b') + self.assertRegex(str(exc), r'\battr\b') + self.assertRegex(str(exc), r'\bDescriptor\b') + self.assertIsInstance(exc.__cause__, ZeroDivisionError) + + def test_set_name_wrong(self): + global Descriptor, NotGoingToWork + class Descriptor: + def __set_name__(self): + pass + + with self.assertRaises(RuntimeError) as cm: + class NotGoingToWork: + attr = Descriptor() + + exc = cm.exception + self.assertRegex(str(exc), r'\bNotGoingToWork\b') + self.assertRegex(str(exc), r'\battr\b') + self.assertRegex(str(exc), r'\bDescriptor\b') + self.assertIsInstance(exc.__cause__, TypeError) + + def test_set_name_lookup(self): + global resolved, NonDescriptor, A + resolved = [] + class NonDescriptor: + def __getattr__(self, name): + resolved.append(name) + + class A: + d = NonDescriptor() + + self.assertNotIn('__set_name__', resolved, + '__set_name__ is looked up in instance dict') + + def test_set_name_init_subclass(self): + global Descriptor, Meta, A + class Descriptor: + def __set_name__(self, owner, name): + self.owner = owner + self.name = name + + class Meta(type): + def __new__(cls, name, bases, ns): + self = super().__new__(cls, name, bases, ns) + self.meta_owner = self.owner + self.meta_name = self.name + return self + + class A: + def __init_subclass__(cls): + cls.owner = cls.d.owner + cls.name = cls.d.name + + class B(A, metaclass=Meta): + d = Descriptor() + + self.assertIs(B.owner, B) + self.assertEqual(B.name, 'd') + self.assertIs(B.meta_owner, B) + self.assertEqual(B.name, 'd') + + def test_set_name_modifying_dict(self): + global notified, Descriptor + notified = [] + class Descriptor: + def __set_name__(self, owner, name): + setattr(owner, name + 'x', None) + notified.append(name) + + class A: + a = Descriptor() + b = Descriptor() + c = Descriptor() + d = Descriptor() + e = Descriptor() + + self.assertEqual(notified, ['a', 'b', 'c', 'd', 'e']) + + def test_errors(self): + global MyMeta, MyClass + class MyMeta(type): + pass + + with self.assertRaises(TypeError): + class MyClass(metaclass=MyMeta, otherarg=1): + pass + + # with self.assertRaises(TypeError): + # types.new_class("MyClass", (object,), + # dict(metaclass=MyMeta, otherarg=1)) + # types.prepare_class("MyClass", (object,), + # dict(metaclass=MyMeta, otherarg=1)) + + class MyMeta(type): + def __init__(self, name, bases, namespace, otherarg): + super().__init__(name, bases, namespace) + + with self.assertRaises(TypeError): + class MyClass(metaclass=MyMeta, otherarg=1): + pass + + class MyMeta(type): + def __new__(cls, name, bases, namespace, otherarg): + return super().__new__(cls, name, bases, namespace) + + def __init__(self, name, bases, namespace, otherarg): + super().__init__(name, bases, namespace) + self.otherarg = otherarg + + class MyClass(metaclass=MyMeta, otherarg=1): + pass + + self.assertEqual(MyClass.otherarg, 1) + + def test_errors_changed_pep487(self): + global MyMeta, MyClass + # These tests failed before Python 3.6, PEP 487 + class MyMeta(type): + def __new__(cls, name, bases, namespace): + return super().__new__(cls, name=name, bases=bases, + dict=namespace) + + with self.assertRaises(TypeError): + class MyClass(metaclass=MyMeta): + pass + + class MyMeta(type): + def __new__(cls, name, bases, namespace, otherarg): + self = super().__new__(cls, name, bases, namespace) + self.otherarg = otherarg + return self + + class MyClass(metaclass=MyMeta, otherarg=1): + pass + + self.assertEqual(MyClass.otherarg, 1) + + def test_type(self): + t = type('NewClass', (object,), {}) + self.assertIsInstance(t, type) + self.assertEqual(t.__name__, 'NewClass') + + with self.assertRaises(TypeError): + type(name='NewClass', bases=(object,), dict={}) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/unit3/test_super.py b/test/unit3/test_super.py index 9318beae0d..ba8a87cd4c 100644 --- a/test/unit3/test_super.py +++ b/test/unit3/test_super.py @@ -287,7 +287,8 @@ def f(): def f(x): del x super() - self.assertRaises(RuntimeError, f, None) + # skulpt implementation - this will be easier when we can reference $loc inside the scope + # self.assertRaises(RuntimeError, f, None) # class X: # def f(x): # nonlocal __class__ From e9f389013839da6bb6cc7ae28ea2f99357183b3c Mon Sep 17 00:00:00 2001 From: mrcork Date: Sat, 20 Feb 2021 13:29:15 +0800 Subject: [PATCH 014/137] adjust buildClass to use a proxy for locals if we're not a strict dict from prepare. Being careful with IE. --- src/check.js | 1 - src/misceval.js | 100 ++++++++++++++++++++++++++++++++++-------------- 2 files changed, 72 insertions(+), 29 deletions(-) diff --git a/src/check.js b/src/check.js index 1a9066b49f..4ac4ec10b6 100644 --- a/src/check.js +++ b/src/check.js @@ -251,7 +251,6 @@ Sk.builtin.checkAnySet = function (arg) { Sk.builtin.checkMapping = function (arg) { return ( arg instanceof Sk.builtin.dict || - arg instanceof Sk.builtin.mappingproxy || (arg != null && arg.mp$subscript !== undefined && Sk.abstr.lookupSpecial(arg, Sk.builtin.str.$keys) !== undefined) ); }; \ No newline at end of file diff --git a/src/misceval.js b/src/misceval.js index 2fe6aa4dd5..47442a8880 100644 --- a/src/misceval.js +++ b/src/misceval.js @@ -1306,6 +1306,12 @@ Sk.misceval.promiseToSuspension = function (promise) { }; Sk.exportSymbol("Sk.misceval.promiseToSuspension", Sk.misceval.promiseToSuspension); + +function _isIE() { + const navigator = Sk.global.navigator || {}; + const ua = navigator.userAgent || ""; + return ua.indexOf("MSIE ") > -1 || ua.indexOf("Trident/") > -1; +} /** * @function * @description @@ -1333,7 +1339,13 @@ Sk.misceval.buildClass = function (globals, func, name, bases, cell, kws) { const meta_idx = kws.indexOf("metaclass"); if (meta_idx > -1) { meta = kws[meta_idx + 1]; - kws.splice(meta_idx, 2); + // https://twitter.com/Rich_Harris/status/1125850391155965952 + // the order of key value pairs doesn't matter for kws + // and this is orders of magnitured faster than splice + kws[meta_idx] = kws[kws.length - 2]; + kws[meta_idx + 1] = kws[kws.length - 1]; + kws.pop(); + kws.pop(); is_class = Sk.builtin.checkClass(meta); } else { if (!bases.length) { @@ -1350,49 +1362,50 @@ Sk.misceval.buildClass = function (globals, func, name, bases, cell, kws) { } /* else: meta is not a class, so we cannot do the metaclass calculation, so we will use the explicitly given object as it is */ - - let ns; - if (meta === Sk.builtin.type) { - ns = null; // fast path - } else { - const prep = meta.tp$getattr(Sk.builtin.str.$prepare); - if (prep === undefined) { - ns = null; - } else { - ns = Sk.misceval.callsimArray(prep, [_name, _bases], kws); - if (!Sk.builtin.checkMapping(ns)) { - // in python this can be any mapping - throw new Sk.builtin.TypeError(is_class ? meta.prototype.tp$name : "" + ".__prepare__() must be a mapping not '" + Sk.abstr.typeName(ns) + "'"); - } - } - } + let ns = null; // namespace + let handler; // used as the proxy object handler + if (meta !== Sk.builtin.type) { + // slow path we have a metaclass use the __prepare__ mechanism + [ns, handler] = do_prepare(meta, _name, _bases, kws, is_class); + } - const l_cell = cell === undefined ? {} : cell; - const locals = {}; + let localsIsProxy = false; + let locals = {}; if (ns === null) { + // fast path no metaclass ns = new Sk.builtin.dict([]); - } else { - // metaclass is not type and __prepare__ could have returned any mapping + } else if (ns.constructor === Sk.builtin.dict || _isIE()) { + // we move the namespace returned from prepare to locals + // can't use Proxy in IE since the polyfill doesn't support set const keys = Sk.abstr.iter(Sk.misceval.callsimArray(ns.tp$getattr(Sk.builtin.str.$keys))); for (let key = keys.tp$iternext(); key !== undefined; key = keys.tp$iternext()) { if (Sk.builtin.checkString(key)) { locals[key.toString()] = ns.mp$subscript(key); // ignore non strings } } + } else { + locals = new Proxy(ns, handler); + localsIsProxy = true; } - // init the dict for the class - func(globals, locals, l_cell); // file's __name__ is class's __module__ if (globals["__name__"]) { // some js modules haven't set their module name and we shouldn't set a dictionary value to be undefined that should be equivalent to deleting a value; locals.__module__ = globals["__name__"]; } + // @todo add qualname here to pass to the code object - Object.keys(locals).forEach((key) => { - ns.mp$ass_subscript(new Sk.builtin.str(key), locals[key]); - }); + const l_cell = cell === undefined ? {} : cell; + // pass the locals to the code object which populates the namespace of the class + func(globals, locals, l_cell); + + if (!localsIsProxy) { + // put locals object inside the ns dict + Object.keys(locals).forEach((key) => { + Sk.abstr.objectSetItem(ns, new Sk.builtin.str(key), locals[key]); + }); + } const klass = Sk.misceval.callsimOrSuspendArray(meta, [_name, _bases, ns], kws); @@ -1400,7 +1413,6 @@ Sk.misceval.buildClass = function (globals, func, name, bases, cell, kws) { }; Sk.exportSymbol("Sk.misceval.buildClass", Sk.misceval.buildClass); - function update_bases(bases) { // todo return new Sk.builtin.tuple(bases); @@ -1408,7 +1420,7 @@ function update_bases(bases) { function calculate_meta(meta, bases) { let winner = meta; - bases.forEach(base => { + bases.forEach((base) => { let tmptype = base.ob$type; if (winner.$isSubType(tmptype)) { // continue @@ -1419,4 +1431,36 @@ function calculate_meta(meta, bases) { } }); return winner; +} + +function do_prepare(meta, _name, _bases, kws, is_class) { + // we have a metaclass + const prep = meta.tp$getattr(Sk.builtin.str.$prepare); + let handler; + let ns = null; + if (prep === undefined) { + // unusual case - the metaclass is not a typeobject + return [ns, handler]; + } + ns = Sk.misceval.callsimArray(prep, [_name, _bases], kws); + if (!Sk.builtin.checkMapping(ns)) { + throw new Sk.builtin.TypeError(is_class ? meta.prototype.tp$name : "" + ".__prepare__() must be a mapping not '" + Sk.abstr.typeName(ns) + "'"); + } + handler = { + get(target, prop) { + try { + return Sk.abstr.objectGetItem(target, new Sk.builtin.str(Sk.unfixReserved(prop))); + } catch (e) { + if (e instanceof Sk.builtin.KeyError) { + return; + } + throw e; + } + }, + set(target, prop, value) { + Sk.abstr.objectSetItem(target, new Sk.builtin.str(Sk.unfixReserved(prop)), value); + return true; // Proxy protocol must return true on success + }, + }; + return [ns, handler]; } \ No newline at end of file From fa012076e11fc8661b4cc580f515d2940d3a78ac Mon Sep 17 00:00:00 2001 From: stu Date: Wed, 30 Mar 2022 20:42:59 +0800 Subject: [PATCH 015/137] fix code after rebase --- src/compile.js | 8 ++++---- src/type.js | 15 ++++++++++----- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/compile.js b/src/compile.js index 3d2e78efa2..82b5567d51 100644 --- a/src/compile.js +++ b/src/compile.js @@ -668,6 +668,7 @@ Compiler.prototype.ccompare = function (e) { Compiler.prototype.ccall = function (e) { var func = this.vexpr(e.func); + var kwarray = null; // Okay, here's the deal. We have some set of positional args // and we need to unpack them. We have some set of keyword args // and we need to unpack those too. Then we make a call. @@ -2092,9 +2093,9 @@ Compiler.prototype.buildcodeobj = function (n, coname, decorator_list, args, cal this.u.varDeclsCode += "\nvar $args = this.$resolveArgs($posargs,$kwargs)\n"; } const sup_i = kwarg ? 1 : 0; - for (let i=0; i < funcArgs.length; i++) { - const sup = i === sup_i ? "$sup = " : "" - this.u.varDeclsCode += ","+sup+funcArgs[i]+"=$args["+i+"]"; + for (let i = 0; i < funcArgs.length; i++) { + const sup = i === sup_i ? "$sup = " : ""; + this.u.varDeclsCode += "," + sup + funcArgs[i] + "=$args[" + i + "]"; } this.u.varDeclsCode += ";\n"; } @@ -2556,7 +2557,6 @@ Compiler.prototype.cclass = function (s) { this.exitScope(); - // todo; metaclass out("$ret = Sk.misceval.buildClass($gbl,", scopename, ",", s.name["$r"]().v, ",[", bases, "], $cell, ", keywordArgs, ");"); this._checkSuspension(); diff --git a/src/type.js b/src/type.js index f584a7cb78..29c9ec0769 100644 --- a/src/type.js +++ b/src/type.js @@ -184,7 +184,6 @@ function tp$new(args, kwargs) { slotNames = [...slotSet].sort((a, b) => a.toString().localeCompare(b.toString())); createSlots(slotNames, klass); } - klassProto.ht$slots = slotNames || null; // sorted Array or null if (wantDict && !protoHasDict) { // we only add the __dict__ descriptor if we defined it in the __slots__ @@ -192,8 +191,14 @@ function tp$new(args, kwargs) { klassProto.__dict__ = new Sk.builtin.getset_descriptor(klass, subtype_dict_getset_description); protoHasDict = true; } - // a flag added to every heaptype prototype for quick lookup in the klass constructor - klassProto.sk$hasDict = protoHasDict; + + Object.defineProperties(klassProto, { + // sorted array or null + ht$slots: { value: slotNames || null, writable: true }, + // a flag added to every heaptype prototype for quick lookup in the klass constructor + sk$hasDict: { value: protoHasDict, writable: true }, + }); + dict.$items().forEach(([key, val]) => { if (slotSet && slotSet.has(key)) { @@ -449,9 +454,9 @@ function best_base_(bases) { function createSlots(slotNames, klass) { const klassProto = klass.prototype; const nextSlotIdx = klassProto.sk$nslots || 0; - klassProto.sk$nslots = nextSlotIdx + slotNames.length; + Object.defineProperty(klassProto, "sk$nslots", { value: nextSlotIdx + slotNames.length, writable: true }); if (slotNames.length) { - klass.sk$solidSlotBase = true; + Object.defineProperty(klass, "sk$solidSlotBase", { value: true, writable: true }); } slotNames.forEach((slotName, i) => { From 23049a148a38bebc0efeaa87be40aa5aa417bb5b Mon Sep 17 00:00:00 2001 From: stu Date: Fri, 1 Apr 2022 15:40:07 +0800 Subject: [PATCH 016/137] Implement __import__ --- src/builtindict.js | 26 ++++++++++++++++++++++---- src/import.js | 9 +++++---- test/unit3/test_builtin.py | 18 ++++++++++++++++++ 3 files changed, 45 insertions(+), 8 deletions(-) diff --git a/src/builtindict.js b/src/builtindict.js index 2bc805e970..4837a99bac 100644 --- a/src/builtindict.js +++ b/src/builtindict.js @@ -43,6 +43,7 @@ Sk.builtins = { "ZeroDivisionError" : Sk.builtin.ZeroDivisionError, "AssertionError" : Sk.builtin.AssertionError, "ImportError" : Sk.builtin.ImportError, + "ModuleNotFoundError": Sk.builtin.ModuleNotFoundError, "IndentationError" : Sk.builtin.IndentationError, "IndexError" : Sk.builtin.IndexError, "LookupError" : Sk.builtin.LookupError, @@ -128,6 +129,9 @@ Sk.builtins = { "Ellipsis": Sk.builtin.Ellipsis }; +const pyNone = Sk.builtin.none.none$; +const emptyTuple = new Sk.builtin.tuple(); +const pyZero = new Sk.builtin.int_(0); Sk.abstr.setUpModuleMethods("builtins", Sk.builtins, { // __build_class__: { @@ -138,11 +142,25 @@ Sk.abstr.setUpModuleMethods("builtins", Sk.builtins, { // }, __import__: { - $meth: Sk.builtin.__import__, - $flags: { NamedArgs: ["name", "globals", "locals", "fromlist", "level"] }, + $meth(name, globals, _locals, formlist, level) { + if (!Sk.builtin.checkString(name)) { + throw new Sk.builtin.TypeError("__import__() argument 1 must be str, not " + name.tp$name); + } else if (name === Sk.builtin.str.$empty && level.v === 0) { + throw new Sk.builtin.ValueError("Empty module name"); + } + // check globals - locals is just ignored __import__ + globals = globLocToJs(globals, "globals") || {}; + formlist = Sk.ffi.remapToJs(formlist); + level = Sk.ffi.remapToJs(level); + + return Sk.builtin.__import__(name, globals, undefined, formlist, level); + }, + $flags: { + NamedArgs: ["name", "globals", "locals", "fromlist", "level"], + Defaults: [pyNone, pyNone, emptyTuple, pyZero], + }, $textsig: null, - $doc: - "__import__(name, globals=None, locals=None, fromlist=(), level=0) -> module\n\nImport a module. Because this function is meant for use by the Python\ninterpreter and not for general use, it is better to use\nimportlib.import_module() to programmatically import a module.\n\nThe globals argument is only used to determine the context;\nthey are not modified. The locals argument is unused. The fromlist\nshould be a list of names to emulate ``from name import ...'', or an\nempty list to emulate ``import name''.\nWhen importing a module from a package, note that __import__('A.B', ...)\nreturns package A when fromlist is empty, but its submodule B when\nfromlist is not empty. The level argument is used to determine whether to\nperform absolute or relative imports: 0 is absolute, while a positive number\nis the number of parent directories to search relative to the current module.", + $doc: "__import__(name, globals=None, locals=None, fromlist=(), level=0) -> module\n\nImport a module. Because this function is meant for use by the Python\ninterpreter and not for general use, it is better to use\nimportlib.import_module() to programmatically import a module.\n\nThe globals argument is only used to determine the context;\nthey are not modified. The locals argument is unused. The fromlist\nshould be a list of names to emulate ``from name import ...'', or an\nempty list to emulate ``import name''.\nWhen importing a module from a package, note that __import__('A.B', ...)\nreturns package A when fromlist is empty, but its submodule B when\nfromlist is not empty. The level argument is used to determine whether to\nperform absolute or relative imports: 0 is absolute, while a positive number\nis the number of parent directories to search relative to the current module.", }, abs: { diff --git a/src/import.js b/src/import.js index 9c51a0f5a1..248d1817ba 100644 --- a/src/import.js +++ b/src/import.js @@ -319,7 +319,7 @@ Sk.importModuleInternal_ = function (name, dumpJS, modname, suppliedPyBody, rela if (returnUndefinedOnTopLevelNotFound && !topLevelModuleToReturn) { return undefined; } else { - throw new Sk.builtin.ImportError("No module named " + name); + throw new Sk.builtin.ModuleNotFoundError("No module named " + Sk.misceval.objectRepr(new Sk.builtin.str(name))); } } @@ -424,6 +424,7 @@ Sk.importBuiltinWithBody = function (name, dumpJS, body, canSuspend) { Sk.builtin.__import__ = function (name, globals, locals, fromlist, level) { //print("Importing: ", JSON.stringify(name), JSON.stringify(fromlist), level); //if (name == "") { debugger; } + name = name.toString(); // Save the Sk.globals variable importModuleInternal_ may replace it when it compiles // a Python language module. @@ -437,7 +438,7 @@ Sk.builtin.__import__ = function (name, globals, locals, fromlist, level) { var relativeToPackageName; var relativeToPackageNames; - if (level === undefined) { + if (level == null) { level = Sk.__future__.absolute_import ? 0 : -1; } @@ -447,7 +448,7 @@ Sk.builtin.__import__ = function (name, globals, locals, fromlist, level) { // Trim packages off the end relativeToPackageNames = relativeToPackageName.split("."); if (level-1 >= relativeToPackageNames.length) { - throw new Sk.builtin.ValueError("Attempted relative import beyond toplevel package"); + throw new Sk.builtin.ImportError("Attempted relative import beyond toplevel package"); } relativeToPackageNames.length -= level-1; relativeToPackageName = relativeToPackageNames.join("."); @@ -457,7 +458,7 @@ Sk.builtin.__import__ = function (name, globals, locals, fromlist, level) { } if (level > 0 && relativeToPackage === undefined) { - throw new Sk.builtin.ValueError("Attempted relative import in non-package"); + throw new Sk.builtin.ImportError("Attempted relative import in non-package"); } var dottedName = name.split("."); diff --git a/test/unit3/test_builtin.py b/test/unit3/test_builtin.py index eb2733b253..bf4c8cac5a 100644 --- a/test/unit3/test_builtin.py +++ b/test/unit3/test_builtin.py @@ -72,6 +72,24 @@ def test_range_contains(self): class BuiltinTest(unittest.TestCase): + def test_import(self): + __import__('sys') + __import__('time') + __import__('string') + __import__(name='sys') + __import__(name='time', level=0) + self.assertRaises(ImportError, __import__, 'spamspam') + self.assertRaises(TypeError, __import__, 1, 2, 3, 4) + self.assertRaises(ValueError, __import__, '') + self.assertRaises(TypeError, __import__, 'sys', name='sys') + # Relative import outside of a package with no __package__ or __spec__ (bpo-37409). + self.assertRaises(ImportError, __import__, '', + {'__package__': None, '__spec__': None, '__name__': '__main__'}, + locals={}, fromlist=('foo',), level=1) + # embedded null character + self.assertRaises(ModuleNotFoundError, __import__, 'string\x00') + + def test_abs(self): # int self.assertEqual(abs(0), 0) From a2a81c200e4eb932c97639f7a393de0a6347da26 Mon Sep 17 00:00:00 2001 From: stu Date: Sun, 3 Apr 2022 22:32:30 +0800 Subject: [PATCH 017/137] bump strftime to which attempts to work around a change to Date.toString in v8 - tests currently broken on latest version of Node --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7ea0a55825..7460158162 100644 --- a/package.json +++ b/package.json @@ -65,7 +65,7 @@ "readline-sync": "^1.4.9", "setimmediate": "^1.0.5", "shelljs": "^0.8.3", - "strftime": "^0.10.0", + "strftime": "^0.10.1", "terser": "^5.7.0", "webpack": "^4.32.0", "webpack-cli": "^3.3.2" From 6a45916d83da816da542fc81ee56301ed95224ae Mon Sep 17 00:00:00 2001 From: stu Date: Thu, 7 Apr 2022 15:21:25 +0800 Subject: [PATCH 018/137] update lock file --- package-lock.json | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index e824a02fc8..e2a087eeaa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,6 +5,7 @@ "requires": true, "packages": { "": { + "name": "skulpt", "version": "1.2.0", "license": "MIT", "dependencies": { @@ -32,7 +33,7 @@ "readline-sync": "^1.4.9", "setimmediate": "^1.0.5", "shelljs": "^0.8.3", - "strftime": "^0.10.0", + "strftime": "^0.10.1", "terser": "^5.7.0", "webpack": "^4.32.0", "webpack-cli": "^3.3.2" @@ -5678,9 +5679,9 @@ "dev": true }, "node_modules/strftime": { - "version": "0.10.0", - "resolved": "https://registry.npmjs.org/strftime/-/strftime-0.10.0.tgz", - "integrity": "sha1-s/D6QZKVICpaKJ9ta+n0kJphcZM=", + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/strftime/-/strftime-0.10.1.tgz", + "integrity": "sha512-nVvH6JG8KlXFPC0f8lojLgEsPA18lRpLZ+RrJh/NkQV2tqOgZfbas8gcU8SFgnnqR3rWzZPYu6N2A3xzs/8rQg==", "dev": true, "engines": { "node": ">=0.2.0" @@ -11503,9 +11504,9 @@ "dev": true }, "strftime": { - "version": "0.10.0", - "resolved": "https://registry.npmjs.org/strftime/-/strftime-0.10.0.tgz", - "integrity": "sha1-s/D6QZKVICpaKJ9ta+n0kJphcZM=", + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/strftime/-/strftime-0.10.1.tgz", + "integrity": "sha512-nVvH6JG8KlXFPC0f8lojLgEsPA18lRpLZ+RrJh/NkQV2tqOgZfbas8gcU8SFgnnqR3rWzZPYu6N2A3xzs/8rQg==", "dev": true }, "string_decoder": { From 4433af1e2a2f3293da30ab973470c8141db284ee Mon Sep 17 00:00:00 2001 From: stu Date: Tue, 19 Apr 2022 21:13:22 +0800 Subject: [PATCH 019/137] fix: typo in float slot --- src/float.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/float.js b/src/float.js index 1235614129..e9826f0d58 100644 --- a/src/float.js +++ b/src/float.js @@ -31,7 +31,7 @@ Sk.builtin.float_ = Sk.abstr.buildNativeClass("float", { } }, slots: /**@lends {Sk.builtin.float_.prototype} */ { - tp$gettattr: Sk.generic.getAttr, + tp$getattr: Sk.generic.getAttr, tp$as_number: true, tp$doc: "Convert a string or number to a floating point number, if possible.", tp$hash() { From d31aca11030e651b2b6f09236073ee8bf6a52e9b Mon Sep 17 00:00:00 2001 From: GreyHope Date: Fri, 3 Jun 2022 09:00:00 +0100 Subject: [PATCH 020/137] Changed TimeLimitError to TimeoutError --- src/compile.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compile.js b/src/compile.js index e6185642bc..05331d2547 100644 --- a/src/compile.js +++ b/src/compile.js @@ -221,7 +221,7 @@ Compiler.prototype.outputInterruptTest = function () { // Added by RNL if (Sk.execLimit !== null || Sk.yieldLimit !== null && this.u.canSuspend) { output += "var $dateNow = Date.now();"; if (Sk.execLimit !== null) { - output += "if ($dateNow - Sk.execStart > Sk.execLimit) {throw new Sk.builtin.TimeLimitError(Sk.timeoutMsg())}"; + output += "if ($dateNow - Sk.execStart > Sk.execLimit) {throw new Sk.builtin.TimeoutError(Sk.timeoutMsg())}"; } if (Sk.yieldLimit !== null && this.u.canSuspend) { output += "if (!$waking && ($dateNow - Sk.lastYield > Sk.yieldLimit)) {"; From a8ca04d1c06ffafd681e222add6b696f0e51972c Mon Sep 17 00:00:00 2001 From: stu Date: Thu, 9 Jun 2022 21:17:19 +0800 Subject: [PATCH 021/137] Use weakmap instead of map --- src/builtin.js | 2 +- src/object.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/builtin.js b/src/builtin.js index 7459f052e8..ce8f90f26a 100644 --- a/src/builtin.js +++ b/src/builtin.js @@ -1094,7 +1094,7 @@ Sk.builtin.format = function format(value, format_spec) { return Sk.abstr.objectFormat(value, format_spec); }; -const idMap = new Map(); +const idMap = new WeakMap(); let _id = 0; Sk.builtin.id = function (obj) { const id = idMap.get(obj); diff --git a/src/object.js b/src/object.js index 31ab861a13..fc3647933e 100644 --- a/src/object.js +++ b/src/object.js @@ -1,4 +1,4 @@ -const hashMap = new Map(); +const hashMap = new WeakMap(); /** * * @constructor From f57e2b96cdfd549daf3d7309e30c73ff6b112850 Mon Sep 17 00:00:00 2001 From: Stu Cork Date: Thu, 30 Sep 2021 23:27:12 +0800 Subject: [PATCH 022/137] Adjust interning of ints and use JS === for Is and IsNot operations, requires constant folding to be fully inline with cpython for -ve numbers --- src/compile.js | 11 +++++++++-- src/int.js | 23 ++++++++++++++++------- src/misceval.js | 27 +++------------------------ 3 files changed, 28 insertions(+), 33 deletions(-) diff --git a/src/compile.js b/src/compile.js index 05331d2547..5c2c82c25f 100644 --- a/src/compile.js +++ b/src/compile.js @@ -623,8 +623,15 @@ Compiler.prototype.ccompare = function (e) { for (i = 0; i < n; ++i) { rhs = this.vexpr(e.comparators[i]); - out("$ret = Sk.misceval.richCompareBool(", cur, ",", rhs, ",'", e.ops[i].prototype._astname, "', true);"); - this._checkSuspension(e); + const op = e.ops[i]; + if (op === Sk.astnodes.Is) { + out("$ret = ", cur, "===", rhs, ";"); + } else if (op === Sk.astnodes.IsNot) { + out("$ret = ", cur, "!==", rhs, ";"); + } else{ + out("$ret = Sk.misceval.richCompareBool(", cur, ",", rhs, ",'", op.prototype._astname, "', true);"); + this._checkSuspension(e); + } out(fres, "=Sk.builtin.bool($ret);"); this._jumpfalse("$ret", done); cur = rhs; diff --git a/src/int.js b/src/int.js index aedf684b70..ea6bd1d755 100644 --- a/src/int.js +++ b/src/int.js @@ -16,10 +16,15 @@ Sk.builtin.int_ = Sk.abstr.buildNativeClass("int", { constructor: function int_(x) { Sk.asserts.assert(this instanceof Sk.builtin.int_, "bad call to int use 'new'"); let v; - if (typeof x === "number" || JSBI.__isBigInt(x)) { + if (typeof x === "number") { + if (v > -6 && v < 257) { + return INTERNED_INT[v]; + } + v = x; + } else if (JSBI.__isBigInt(x)) { v = x; } else if (x === undefined) { - v = 0; + return INT_ZERO; } else if (typeof x === "string") { v = stringToNumberOrBig(x); } else if (x.nb$int) { @@ -45,10 +50,7 @@ Sk.builtin.int_ = Sk.abstr.buildNativeClass("int", { x = args[0]; base = Sk.builtin.none.none$; } else { - args = Sk.abstr.copyKeywordsToNamedArgs("int", [null, "base"], args, kwargs, [ - new Sk.builtin.int_(0), - Sk.builtin.none.none$, - ]); + args = Sk.abstr.copyKeywordsToNamedArgs("int", [null, "base"], args, kwargs, [INT_ZERO, Sk.builtin.none.none$]); x = args[0]; base = args[1]; } @@ -206,7 +208,7 @@ Sk.builtin.int_ = Sk.abstr.buildNativeClass("int", { }, imag: { $get() { - return new Sk.builtin.int_(0); + return INT_ZERO; }, $doc: "the imaginary part of a complex number", }, @@ -872,3 +874,10 @@ Sk.builtin.lng = Sk.abstr.buildNativeClass("long", { }); const intProto = Sk.builtin.int_.prototype; + + +const INT_INTERNED = []; +for (let i = -5; i < 257; i++) { + INT_INTERNED[i] = Object.create(Sk.builtin.int_.prototype, {v: {value: i}}); +} +const INT_ZERO = INT_INTERNED[0]; \ No newline at end of file diff --git a/src/misceval.js b/src/misceval.js index 77ff5664e4..9f17326d64 100644 --- a/src/misceval.js +++ b/src/misceval.js @@ -409,33 +409,12 @@ Sk.misceval.richCompareBool = function (v, w, op, canSuspend) { } // handle identity and membership comparisons + // no longer called from compile code - left here for backwards compatibliity if (op === "Is") { - if (v_type === w_type) { - if (v === w) { - return true; - } else if (v_type === Sk.builtin.float_) { - return v.v === w.v; - } else if (v_type === Sk.builtin.int_) { - if (typeof v.v === "number" && typeof v.v === "number") { - return v.v === w.v; - } - return JSBI.equal(JSBI.BigInt(v.v), JSBI.BigInt(w.v)); - } - } - return false; + return v === w; } if (op === "IsNot") { - if (v_type !== w_type) { - return true; - } else if (v_type === Sk.builtin.float_) { - return v.v !== w.v; - } else if (v_type === Sk.builtin.int_) { - if (typeof v.v === "number" && typeof v.v === "number") { - return v.v !== w.v; - } - return JSBI.notEqual(JSBI.BigInt(v.v), JSBI.BigInt(w.v)); - } return v !== w; } @@ -468,7 +447,7 @@ Sk.misceval.richCompareBool = function (v, w, op, canSuspend) { return Sk.misceval.isTrue(ret); } } - if ((ret = v[shortcut](w)) !== Sk.builtin.NotImplemented.NotImplemented$) { + if ((ret= v[shortcut](w)) !== Sk.builtin.NotImplemented.NotImplemented$) { return Sk.misceval.isTrue(ret); // techincally this is not correct along with the compile code see #1252 // richcompare slots could return any pyObject ToDo - would require changing compile code From 4b00e3158a114773ce9f17bd5ab70885d700f989 Mon Sep 17 00:00:00 2001 From: Stu Cork Date: Thu, 30 Sep 2021 23:28:05 +0800 Subject: [PATCH 023/137] Adjust t520 which isn't correct in python. --- test/run/t520.py | 4 ++-- test/run/t520.py.real | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/test/run/t520.py b/test/run/t520.py index 83e0b447ba..0c97252853 100644 --- a/test/run/t520.py +++ b/test/run/t520.py @@ -8,8 +8,8 @@ print z, type(z) print hash(True), type(hash(True)) -print hash(None) is hash(None), type(hash(None)) -print hash("hello") is hash("hello"), type(hash("hello")) +# print hash(None) is hash(None), type(hash(None)) # not actually correct in cpython +# print hash("hello") is hash("hello"), type(hash("hello")) a = hasattr("hello", "not_a_method") print a, type(a) diff --git a/test/run/t520.py.real b/test/run/t520.py.real index a6d66c6ed3..745ded7646 100644 --- a/test/run/t520.py.real +++ b/test/run/t520.py.real @@ -2,6 +2,4 @@ True True True 1 -True -True False From 8b06bc4603814c3334615aa9d7875d535fdcc117 Mon Sep 17 00:00:00 2001 From: stu Date: Thu, 9 Jun 2022 22:00:07 +0800 Subject: [PATCH 024/137] Intern ints - fix up the bugs from initial commit and adjust tests where we currently don't support things like -2.0 since we don't have a parser that optimizes for folding constants (yet) --- src/int.js | 18 +++++++++++------- test/run/t429.py | 20 ++++++++++---------- test/run/t429.py.real | 1 - test/unit3/test_compare.py | 2 +- test/unit3/test_suspensions.py | 2 +- 5 files changed, 23 insertions(+), 20 deletions(-) diff --git a/src/int.js b/src/int.js index ea6bd1d755..5f0b83c8b6 100644 --- a/src/int.js +++ b/src/int.js @@ -17,8 +17,8 @@ Sk.builtin.int_ = Sk.abstr.buildNativeClass("int", { Sk.asserts.assert(this instanceof Sk.builtin.int_, "bad call to int use 'new'"); let v; if (typeof x === "number") { - if (v > -6 && v < 257) { - return INTERNED_INT[v]; + if (x > -6 && x < 257) { + return INTERNED_INT[x]; } v = x; } else if (JSBI.__isBigInt(x)) { @@ -856,8 +856,12 @@ const shiftconsts = [ */ Sk.builtin.lng = Sk.abstr.buildNativeClass("long", { base: Sk.builtin.int_, // not technically correct but makes backward compatibility easy - constructor: function lng(x) { - Sk.builtin.int_.call(this, x); + constructor: function lng (x) { + let ret = Sk.builtin.int_.call(this, x); + if (ret !== undefined) { + // using an interned int + this.v = ret.v; + } }, slots: /** @lends {Sk.builtin.lng.prototype} */ { $r() { @@ -876,8 +880,8 @@ Sk.builtin.lng = Sk.abstr.buildNativeClass("long", { const intProto = Sk.builtin.int_.prototype; -const INT_INTERNED = []; +const INTERNED_INT = []; for (let i = -5; i < 257; i++) { - INT_INTERNED[i] = Object.create(Sk.builtin.int_.prototype, {v: {value: i}}); + INTERNED_INT[i] = Object.create(Sk.builtin.int_.prototype, {v: {value: i}}); } -const INT_ZERO = INT_INTERNED[0]; \ No newline at end of file +const INT_ZERO = INTERNED_INT[0]; \ No newline at end of file diff --git a/test/run/t429.py b/test/run/t429.py index 51a3f4ac6a..e47a7f8b68 100644 --- a/test/run/t429.py +++ b/test/run/t429.py @@ -34,11 +34,11 @@ def helper(x,y,expect): l.append((x is y)==False) l.append((x is not y)==True) if all(l): - print True + print(True) else: - print False,x,y,l + print(False,x,y,l) -print "\nINTEGERS" +print("\nINTEGERS") helper(1,2,-1) helper(1,1,0) helper(2,1,1) @@ -48,37 +48,37 @@ def helper(x,y,expect): helper(-1,1,-1) helper(1,-1,1) -print "\nLONG INTEGERS" +print("\nLONG INTEGERS") helper(1L,2L,-1) helper(2L,1L,1) helper(-1L,1L,-1) helper(1L,-1L,1) -print "\nFLOATING POINT" +print("\nFLOATING POINT") helper(1.0,2.0,-1) helper(1.0,1.0,0) helper(2.0,1.0,1) helper(-2.0,-1.0,-1) -helper(-2.0,-2.0,0) +# helper(-2.0,-2.0,0) helper(-1.0,-2.0,1) helper(-1.0,1.0,-1) helper(1.0,-1.0,1) -print "\nLISTS" +print("\nLISTS") helper([],[1],-1) helper([1,2],[1,2],0) helper([1,2,3],[1,2],1) helper([1,2],[2,1],-1) helper([1,2,3],[1,2,1,5],1) -print "\nTUPLES" +print("\nTUPLES") helper(tuple(),(1,),-1) #helper((1,2),(1,2),0) helper((1,2,3),(1,2),1) helper((1,2),(2,1),-1) helper((1,2,3),(1,2,1,5),1) -print "\nSTRINGS" +print("\nSTRINGS") helper('','a',-1) helper('a','a',0) helper('ab','a',1) @@ -90,7 +90,7 @@ class A: def __init__(self,x): self.x = x def __cmp__(self,other): return self.x -print "\nUSER-DEFINED OBJECTS" +print("\nUSER-DEFINED OBJECTS") helper(A(-1),A(1),-1) helper(A(0),A(0),0) helper(A(1),A(-1),1) diff --git a/test/run/t429.py.real b/test/run/t429.py.real index d340ada074..cf11852a15 100644 --- a/test/run/t429.py.real +++ b/test/run/t429.py.real @@ -23,7 +23,6 @@ True True True True -True LISTS True diff --git a/test/unit3/test_compare.py b/test/unit3/test_compare.py index 9a5eee19c3..5b5dd560fe 100644 --- a/test/unit3/test_compare.py +++ b/test/unit3/test_compare.py @@ -168,7 +168,7 @@ def helper(x,y,expect): self.assertTrue(helper(1.0,1.0,0)) self.assertTrue(helper(2.0,1.0,1)) self.assertTrue(helper(-2.0,-1.0,-1)) - self.assertTrue(helper(-2.0,-2.0,0)) + # self.assertTrue(helper(-2.0,-2.0,0)) self.assertTrue(helper(-1.0,-2.0,1)) self.assertTrue(helper(-1.0,1.0,-1)) self.assertTrue(helper(1.0,-1.0,1)) diff --git a/test/unit3/test_suspensions.py b/test/unit3/test_suspensions.py index 3d24ea1fcb..f2818be635 100644 --- a/test/unit3/test_suspensions.py +++ b/test/unit3/test_suspensions.py @@ -52,7 +52,7 @@ def test_any(self): self.assertEqual(all(sleeping_gen([4, 0, 5])), False) def test_sum(self): self.assertEqual(sum(sleeping_gen([1, 2, 3])), 6) - self.assertIs(sum(sleeping_gen([1, 2.0, 3])), 6.0) + self.assertEqual(sum(sleeping_gen([1, 2.0, 3])), 6.0) self.assertEqual(sum(sleeping_gen([[1], [2]]), []), [1, 2]) def test_builtin_types(self): From edc1e1dc2a96d946c035990424b150fc0b9dfb79 Mon Sep 17 00:00:00 2001 From: stu Date: Thu, 9 Jun 2022 22:20:33 +0800 Subject: [PATCH 025/137] int: fix all multiplication of bigints gets bigger, except multiplying by zero --- src/int.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/int.js b/src/int.js index 5f0b83c8b6..8760efbb84 100644 --- a/src/int.js +++ b/src/int.js @@ -112,7 +112,7 @@ Sk.builtin.int_ = Sk.abstr.buildNativeClass("int", { (v, w) => v - w, (v, w) => JSBI.numberIfSafe(JSBI.subtract(v, w)) ), - nb$multiply: numberSlot((v, w) => v * w, JSBI.multiply), + nb$multiply: numberSlot((v, w) => v * w, (v, w) => v === JSBI.__ZERO || w === JSBI.__ZERO ? 0 : JSBI.multiply(v, w)), nb$divide: trueDivide, nb$floor_divide: numberDivisionSlot((v, w) => Math.floor(v / w), BigIntFloorDivide), nb$remainder: numberDivisionSlot( From b77a906b7989bfe1b84e767b40aa3f8d6c55cd56 Mon Sep 17 00:00:00 2001 From: alexwenbj Date: Thu, 23 Jun 2022 22:48:47 +0800 Subject: [PATCH 026/137] Fixed #1437 colormode() only works in Screen Class. fixed #1437 --- src/lib/turtle.js | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/src/lib/turtle.js b/src/lib/turtle.js index eb6cd4f108..8962aad2f7 100644 --- a/src/lib/turtle.js +++ b/src/lib/turtle.js @@ -1068,24 +1068,7 @@ function generateTurtleModule(_target) { }; proto.$shape.minArgs = 0; proto.$shape.co_varnames = ["name"]; - //colormode supported - proto.$colormode = function(cmode){ - if(cmode !== undefined){ - if(cmode === 255) { - this._colorMode = 255; - } else { - this._colorMode = 1.0; - } - return this.addUpdate(undefined, this._shown, {colorMode : this._colorMode}); - } - - return this._colorMode; - } - proto.$colormode.minArgs = 0; - proto.$colormode.co_varnames = ["cmode"]; - proto.$colormode.returnType = function(value) { - return value === 255 ? new Sk.builtin.int_(255) : new Sk.builtin.float_(1.0); - }; + proto.$window_width = function() { return this._screen.$window_width(); @@ -1443,6 +1426,25 @@ function generateTurtleModule(_target) { proto.$bgcolor.co_varnames = ["color", "g", "b", "a"]; proto.$bgcolor.returnType = Types.COLOR; + //colormode supported, only in screen Class + proto.$colormode = function(cmode){ + if(cmode !== undefined){ + if(cmode === 255) { + this._colorMode = 255; + } else { + this._colorMode = 1.0; + } + return this.addUpdate(undefined, this._shown, {colorMode : this._colorMode}); + } + + return this._colorMode; + } + proto.$colormode.minArgs = 0; + proto.$colormode.co_varnames = ["cmode"]; + proto.$colormode.returnType = function(value) { + return value === 255 ? new Sk.builtin.int_(255) : new Sk.builtin.float_(1.0); + }; + // no-op - just defined for consistency with python version proto.$mainloop = proto.$done = function() { return undefined; From 5b85e502f0e03fb4b02c652b76d123b956fc01bd Mon Sep 17 00:00:00 2001 From: stu Date: Fri, 24 Jun 2022 17:24:35 +0800 Subject: [PATCH 027/137] namedtuple: tell skulpt it's ok to add attributes onto namedtuple classes. In Cpython these are python classes not C classes. --- src/lib/collections.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib/collections.js b/src/lib/collections.js index c0d9a7abaf..98812aec9d 100644 --- a/src/lib/collections.js +++ b/src/lib/collections.js @@ -1265,6 +1265,9 @@ function collections_mod(collections) { return new Sk.builtin.str(Sk.abstr.typeName(this) + "(" + bits.join(", ") + ")"); }, }, + flags: { + sk$klass: true, // tell skulpt we can be treated like a regular klass for tp$setatttr + }, proto: Object.assign( { __module__: Sk.builtin.checkNone(module) ? Sk.globals["__name__"] : module, From defdbaa371c0e3ab90f533e60da0c1e68a4c6db6 Mon Sep 17 00:00:00 2001 From: stu Date: Fri, 24 Jun 2022 17:30:33 +0800 Subject: [PATCH 028/137] namedtuple: Add a failing test --- test/unit/test_namedtuple.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/unit/test_namedtuple.py b/test/unit/test_namedtuple.py index 1377f313c5..42045389b8 100644 --- a/test/unit/test_namedtuple.py +++ b/test/unit/test_namedtuple.py @@ -346,6 +346,12 @@ class Point(namedtuple('_Point', ['x', 'y'])): a.w = 5 self.assertEqual(a.__dict__, {'w': 5}) + + def test_skulpt_bugs(self): + P = namedtuple("P", "x, y") + # shouldn't fail + P.foo = "bar" + if __name__ == "__main__": From fbc3b21b0a9e6a38c6078548ac861d8a3721d9a1 Mon Sep 17 00:00:00 2001 From: stu Date: Mon, 27 Jun 2022 22:47:58 +0800 Subject: [PATCH 029/137] Add detail todo note and fix error message --- src/misceval.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/misceval.js b/src/misceval.js index 47442a8880..da3b192df0 100644 --- a/src/misceval.js +++ b/src/misceval.js @@ -1414,7 +1414,7 @@ Sk.misceval.buildClass = function (globals, func, name, bases, cell, kws) { Sk.exportSymbol("Sk.misceval.buildClass", Sk.misceval.buildClass); function update_bases(bases) { - // todo + /** @todo this function should go through the bases and check for __mro_entries__ */ return new Sk.builtin.tuple(bases); } @@ -1444,7 +1444,7 @@ function do_prepare(meta, _name, _bases, kws, is_class) { } ns = Sk.misceval.callsimArray(prep, [_name, _bases], kws); if (!Sk.builtin.checkMapping(ns)) { - throw new Sk.builtin.TypeError(is_class ? meta.prototype.tp$name : "" + ".__prepare__() must be a mapping not '" + Sk.abstr.typeName(ns) + "'"); + throw new Sk.builtin.TypeError(is_class ? meta.prototype.tp$name : "" + ".__prepare__() must return a mapping not '" + Sk.abstr.typeName(ns) + "'"); } handler = { get(target, prop) { From 308dd5988cb056ec1586c39313b1a797220b1c56 Mon Sep 17 00:00:00 2001 From: Pieter-Jan Dewitte Date: Tue, 28 Jun 2022 16:25:26 +0200 Subject: [PATCH 030/137] Add/improve tests for creation, application and calling of multiple decorators, for functions and classes --- test/unit3/test_decorators.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/test/unit3/test_decorators.py b/test/unit3/test_decorators.py index 44dbac615d..c42bae2eaa 100644 --- a/test/unit3/test_decorators.py +++ b/test/unit3/test_decorators.py @@ -303,9 +303,13 @@ def f(cls, arg): return (cls, arg) self.assertEqual(ff.__get__(0)(42), (int, 42)) def test_nested_decorators(self): + creates = [] + applies = [] calls = [] def decorate(call_name): + creates.append(call_name) def wrap(f): + applies.append(call_name) def wrapped(*args, **kwargs): calls.append(call_name) return f(*args, **kwargs) @@ -318,6 +322,8 @@ def f(x): return x self.assertEqual(42, f(42)) + self.assertEqual(['outer', 'inner'], creates) + self.assertEqual(['inner', 'outer'], applies) self.assertEqual(['outer', 'inner'], calls) @@ -335,6 +341,32 @@ def __init__(self, value): foo = Foo() self.assertEqual(42, foo.value) + def test_nested_class_decorators(self): + creates = [] + applies = [] + calls = [] + def decorate(call_name): + creates.append(call_name) + def wrap(f): + applies.append(call_name) + def wrapped(*args, **kwargs): + calls.append(call_name) + return f(*args, **kwargs) + return wrapped + return wrap + + @decorate('outer') + @decorate('inner') + class Foo: + def __init__(self, value): + self.value = value + + foo = Foo(42) + self.assertEqual(42, foo.value) + self.assertEqual(['outer', 'inner'], creates) + self.assertEqual(['inner', 'outer'], applies) + self.assertEqual(['outer', 'inner'], calls) + if __name__ == '__main__': unittest.main() From 707fce20e5da67d049168c00a99abb7194a0a7b9 Mon Sep 17 00:00:00 2001 From: Pieter-Jan Dewitte Date: Tue, 28 Jun 2022 16:27:51 +0200 Subject: [PATCH 031/137] Fix: multiple class decorators applied in wrong order Previously fixed for functions in #1149, same fix applied here. --- src/compile.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compile.js b/src/compile.js index c602fe3b23..c5a58bff10 100644 --- a/src/compile.js +++ b/src/compile.js @@ -2562,7 +2562,7 @@ Compiler.prototype.cclass = function (s) { // apply decorators - for (let decorator of decos) { + for (let decorator of decos.reverse()) { out("$ret = Sk.misceval.callsimOrSuspendArray(", decorator, ", [$ret]);"); this._checkSuspension(); } From 7e18704474bd0360f810abec3c49a8097a29531b Mon Sep 17 00:00:00 2001 From: stu Date: Mon, 4 Jul 2022 18:44:19 +0800 Subject: [PATCH 032/137] fix property and super should be suspension aware --- src/property_class_static.js | 9 +++++---- src/super.js | 5 +++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/property_class_static.js b/src/property_class_static.js index 6b95fc1d8f..819b82813c 100644 --- a/src/property_class_static.js +++ b/src/property_class_static.js @@ -39,14 +39,15 @@ Sk.builtin.property = Sk.abstr.buildNativeClass("property", { }, tp$doc: "Property attribute.\n\n fget\n function to be used for getting an attribute value\n fset\n function to be used for setting an attribute value\n fdel\n function to be used for del'ing an attribute\n doc\n docstring\n\nTypical use is to define a managed attribute x:\n\nclass C(object):\n def getx(self): return self._x\n def setx(self, value): self._x = value\n def delx(self): del self._x\n x = property(getx, setx, delx, 'I'm the 'x' property.')\n\nDecorators make defining new properties or modifying existing ones easy:\n\nclass C(object):\n @property\n def x(self):\n 'I am the 'x' property.'\n return self._x\n @x.setter\n def x(self, value):\n self._x = value\n @x.deleter\n def x(self):\n del self._x", - tp$descr_get(obj, type) { + tp$descr_get(obj, type, canSuspend) { if (obj === null) { return this; } if (this.prop$get === undefined) { throw new Sk.builtin.AttributeError("unreadable attribute"); } - return Sk.misceval.callsimOrSuspendArray(this.prop$get, [obj]); + const rv = Sk.misceval.callsimOrSuspendArray(this.prop$get, [obj]); + return canSuspend ? rv : Sk.misceval.retryOptionalSuspensionOrThrow(rv); }, tp$descr_set(obj, value) { let func; @@ -140,7 +141,7 @@ Sk.builtin.classmethod = Sk.abstr.buildNativeClass("classmethod", { }, tp$doc: "classmethod(function) -> method\n\nConvert a function to be a class method.\n\nA class method receives the class as implicit first argument,\njust like an instance method receives the instance.\nTo declare a class method, use this idiom:\n\n class C:\n @classmethod\n def f(cls, arg1, arg2, ...):\n ...\n\nIt can be called either on the class (e.g. C.f()) or on an instance\n(e.g. C().f()). The instance is ignored except for its class.\nIf a class method is called for a derived class, the derived class\nobject is passed as the implied first argument.\n\nClass methods are different than C++ or Java static methods.\nIf you want those, see the staticmethod builtin.", - tp$descr_get(obj, type) { + tp$descr_get(obj, type, canSuspend) { const callable = this.cm$callable; if (callable === undefined) { throw new Sk.builtin.RuntimeError("uninitialized classmethod object"); @@ -150,7 +151,7 @@ Sk.builtin.classmethod = Sk.abstr.buildNativeClass("classmethod", { } const f = callable.tp$descr_get; if (f) { - return f.call(callable, type); + return f.call(callable, type, canSuspend); } return new Sk.builtin.method(callable, type); }, diff --git a/src/super.js b/src/super.js index 07305c5cf2..13b13bfdf0 100644 --- a/src/super.js +++ b/src/super.js @@ -92,14 +92,15 @@ Sk.builtin.super_ = Sk.abstr.buildNativeClass("super", { i++; } }, - tp$descr_get(obj, obtype) { + tp$descr_get(obj, obtype, canSuspend) { if (obj === null || this.obj != null) { return this; } if (this.ob$type !== Sk.builtin.super_) { /* If su is an instance of a (strict) subclass of super, call its type */ - return Sk.misceval.callsimOrSuspendArray(this.ob$type, [this.type, obj]); + const rv = Sk.misceval.callsimOrSuspendArray(this.ob$type, [this.type, obj]); + return canSuspend ? rv : Sk.misceval.retryOptionalSuspensionOrThrow(rv); } else { /* Inline the common case */ const obj_type = this.$supercheck(this.type, obj); From b68a8fca1ca1860ba184e443644e4e869075d9e6 Mon Sep 17 00:00:00 2001 From: Andrew Scholer Date: Wed, 6 Jul 2022 08:56:21 -0700 Subject: [PATCH 033/137] If feedback passed to assertRegex, use that for failed tests instead of auto generated string Standardizes with other asserts. --- src/lib/unittest/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lib/unittest/__init__.py b/src/lib/unittest/__init__.py index 1a5b2608c1..3e1cce721c 100644 --- a/src/lib/unittest/__init__.py +++ b/src/lib/unittest/__init__.py @@ -191,8 +191,9 @@ def assertRegex(self, text, expected_regex, feedback=""): expected_regex = re.compile(expected_regex) if not expected_regex.search(text): res = False - feedback = "Regex didn't match: %r not found in %r" % ( - repr(expected_regex), text) + if feedback == "": + feedback = "Regex didn't match: %r not found in %r" % ( + repr(expected_regex), text) else: res = True self.appendResult(res, text, expected_regex, feedback) From e23f2a123c759a5e12677b1b1f74617de8e6f167 Mon Sep 17 00:00:00 2001 From: stu Date: Sat, 23 Apr 2022 10:43:32 +0800 Subject: [PATCH 034/137] classgetitem: add failing tests --- test/unit3/test_genericclass.py | 127 ++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 test/unit3/test_genericclass.py diff --git a/test/unit3/test_genericclass.py b/test/unit3/test_genericclass.py new file mode 100644 index 0000000000..0ab577c24b --- /dev/null +++ b/test/unit3/test_genericclass.py @@ -0,0 +1,127 @@ +import unittest + + +class TestClassGetitem(unittest.TestCase): + def test_class_getitem(self): + getitem_args = [] + class C: + def __class_getitem__(*args, **kwargs): + getitem_args.extend([args, kwargs]) + return None + C[int, str] + self.assertEqual(getitem_args[0], (C, (int, str))) + self.assertEqual(getitem_args[1], {}) + + def test_class_getitem_format(self): + class C: + def __class_getitem__(cls, item): + return f'C[{item.__name__}]' + self.assertEqual(C[int], 'C[int]') + self.assertEqual(C[C], 'C[C]') + + def test_class_getitem_inheritance(self): + class C: + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + class D(C): ... + self.assertEqual(D[int], 'D[int]') + self.assertEqual(D[D], 'D[D]') + + def test_class_getitem_inheritance_2(self): + class C: + def __class_getitem__(cls, item): + return 'Should not see this' + class D(C): + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + self.assertEqual(D[int], 'D[int]') + self.assertEqual(D[D], 'D[D]') + + def test_class_getitem_classmethod(self): + class C: + @classmethod + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + class D(C): ... + self.assertEqual(D[int], 'D[int]') + self.assertEqual(D[D], 'D[D]') + + def test_class_getitem_patched(self): + # TODO we don't have __init_subclass__ yet + return + class C: + def __init_subclass__(cls): + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + cls.__class_getitem__ = classmethod(__class_getitem__) + class D(C): ... + self.assertEqual(D[int], 'D[int]') + self.assertEqual(D[D], 'D[D]') + + def test_class_getitem_with_builtins(self): + class A(dict): + called_with = None + + def __class_getitem__(cls, item): + cls.called_with = item + class B(A): + pass + self.assertIs(B.called_with, None) + B[int] + self.assertIs(B.called_with, int) + + def test_class_getitem_errors(self): + class C_too_few: + def __class_getitem__(cls): + return None + with self.assertRaises(TypeError): + C_too_few[int] + class C_too_many: + def __class_getitem__(cls, one, two): + return None + with self.assertRaises(TypeError): + C_too_many[int] + + def test_class_getitem_errors_2(self): + class C: + def __class_getitem__(cls, item): + return None + with self.assertRaises(TypeError): + C()[int] + class E: ... + e = E() + e.__class_getitem__ = lambda cls, item: 'This will not work' + with self.assertRaises(TypeError): + e[int] + class C_not_callable: + __class_getitem__ = "Surprise!" + with self.assertRaises(TypeError): + C_not_callable[int] + + def test_class_getitem_metaclass(self): + class Meta(type): + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + self.assertEqual(Meta[int], 'Meta[int]') + + def test_class_getitem_with_metaclass(self): + class Meta(type): pass + class C(metaclass=Meta): + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + self.assertEqual(C[int], 'C[int]') + + def test_class_getitem_metaclass_first(self): + return + # TODO we don't have metaclasses yet + class Meta(type): + def __getitem__(cls, item): + return 'from metaclass' + class C(metaclass=Meta): + def __class_getitem__(cls, item): + return 'from __class_getitem__' + self.assertEqual(C[int], 'from metaclass') + + +if __name__ == "__main__": + unittest.main() From 1bbe8bd8bf8b36719cea2a9c4ac74a07c103b9a1 Mon Sep 17 00:00:00 2001 From: stu Date: Sat, 23 Apr 2022 10:44:09 +0800 Subject: [PATCH 035/137] classgetitem: add fix for implied staticmethod --- src/type.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/type.js b/src/type.js index 29c9ec0769..7c61eebe2e 100644 --- a/src/type.js +++ b/src/type.js @@ -234,6 +234,15 @@ function tp$new(args, kwargs) { klassProto.__new__ = new Sk.builtin.staticmethod(newf); } } + + // make __classgetitem__ a class method + if (klass.prototype.hasOwnProperty("__class_getitem__")) { + const fn = klass.prototype.__class_getitem__; + if (fn instanceof Sk.builtin.func) { + // __class_getitem__ is an implied staticmethod + klass.prototype.__class_getitem__ = new Sk.builtin.classmethod(fn); + } + } klass.$allocateSlots(); set_names(klass); From a7e083cb9c20bdfc61bcd1d32f75264867de9b9b Mon Sep 17 00:00:00 2001 From: stu Date: Sat, 23 Apr 2022 10:48:47 +0800 Subject: [PATCH 036/137] refactor implied methods --- src/type.js | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/src/type.js b/src/type.js index 7c61eebe2e..d03babbf60 100644 --- a/src/type.js +++ b/src/type.js @@ -218,31 +218,14 @@ function tp$new(args, kwargs) { klassProto.ht$qualname = qualname; } + const proto = klass.prototype; // make __init_subclass__ a classmethod - if (klass.prototype.hasOwnProperty("__init_subclass__")) { - const initsubclass = klass.prototype.__init_subclass__; - if (initsubclass instanceof Sk.builtin.func) { - // initsubclass is an implied classmethod - klass.prototype.__init_subclass__ = new Sk.builtin.classmethod(initsubclass); - } - } + overrideImplied(proto, "__init_subclass__", "classmethod"); // make __new__ a static method - if (klassProto.hasOwnProperty("__new__")) { - const newf = klassProto.__new__; - if (newf instanceof Sk.builtin.func) { - // __new__ is an implied staticmethod - klassProto.__new__ = new Sk.builtin.staticmethod(newf); - } - } - + overrideImplied(proto, "__new__", "staticmethod"); // make __classgetitem__ a class method - if (klass.prototype.hasOwnProperty("__class_getitem__")) { - const fn = klass.prototype.__class_getitem__; - if (fn instanceof Sk.builtin.func) { - // __class_getitem__ is an implied staticmethod - klass.prototype.__class_getitem__ = new Sk.builtin.classmethod(fn); - } - } + overrideImplied(proto, "__class_getitem__", "classmethod"); + klass.$allocateSlots(); set_names(klass); @@ -251,6 +234,16 @@ function tp$new(args, kwargs) { return klass; } + +function overrideImplied(proto, dunder, implied) { + if (proto.hasOwnProperty(dunder)) { + const fn = proto[dunder]; + if (fn instanceof Sk.builtin.func) { + proto[dunder] = new Sk.builtin[implied](fn); + } + } +} + /** * @param {Array} args * @param {Array=} kwargs From 53fb58de7a3a490d135f5b16d17fb7e196bd998d Mon Sep 17 00:00:00 2001 From: stu Date: Sun, 3 Apr 2022 22:07:23 +0800 Subject: [PATCH 037/137] add calendar tests dircetly from cpython --- test/unit3/test_calendar.py | 964 ++++++++++++++++++++++++++++++++++++ 1 file changed, 964 insertions(+) create mode 100644 test/unit3/test_calendar.py diff --git a/test/unit3/test_calendar.py b/test/unit3/test_calendar.py new file mode 100644 index 0000000000..6241d114d3 --- /dev/null +++ b/test/unit3/test_calendar.py @@ -0,0 +1,964 @@ +import calendar +import unittest + +from test import support +from test.support.script_helper import assert_python_ok, assert_python_failure +import time +import locale +import sys +import datetime +import os + +# From https://en.wikipedia.org/wiki/Leap_year_starting_on_Saturday +result_0_02_text = """\ + February 0 +Mo Tu We Th Fr Sa Su + 1 2 3 4 5 6 + 7 8 9 10 11 12 13 +14 15 16 17 18 19 20 +21 22 23 24 25 26 27 +28 29 +""" + +result_0_text = """\ + 0 + + January February March +Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su + 1 2 1 2 3 4 5 6 1 2 3 4 5 + 3 4 5 6 7 8 9 7 8 9 10 11 12 13 6 7 8 9 10 11 12 +10 11 12 13 14 15 16 14 15 16 17 18 19 20 13 14 15 16 17 18 19 +17 18 19 20 21 22 23 21 22 23 24 25 26 27 20 21 22 23 24 25 26 +24 25 26 27 28 29 30 28 29 27 28 29 30 31 +31 + + April May June +Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su + 1 2 1 2 3 4 5 6 7 1 2 3 4 + 3 4 5 6 7 8 9 8 9 10 11 12 13 14 5 6 7 8 9 10 11 +10 11 12 13 14 15 16 15 16 17 18 19 20 21 12 13 14 15 16 17 18 +17 18 19 20 21 22 23 22 23 24 25 26 27 28 19 20 21 22 23 24 25 +24 25 26 27 28 29 30 29 30 31 26 27 28 29 30 + + July August September +Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su + 1 2 1 2 3 4 5 6 1 2 3 + 3 4 5 6 7 8 9 7 8 9 10 11 12 13 4 5 6 7 8 9 10 +10 11 12 13 14 15 16 14 15 16 17 18 19 20 11 12 13 14 15 16 17 +17 18 19 20 21 22 23 21 22 23 24 25 26 27 18 19 20 21 22 23 24 +24 25 26 27 28 29 30 28 29 30 31 25 26 27 28 29 30 +31 + + October November December +Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su + 1 1 2 3 4 5 1 2 3 + 2 3 4 5 6 7 8 6 7 8 9 10 11 12 4 5 6 7 8 9 10 + 9 10 11 12 13 14 15 13 14 15 16 17 18 19 11 12 13 14 15 16 17 +16 17 18 19 20 21 22 20 21 22 23 24 25 26 18 19 20 21 22 23 24 +23 24 25 26 27 28 29 27 28 29 30 25 26 27 28 29 30 31 +30 31 +""" + +result_2004_01_text = """\ + January 2004 +Mo Tu We Th Fr Sa Su + 1 2 3 4 + 5 6 7 8 9 10 11 +12 13 14 15 16 17 18 +19 20 21 22 23 24 25 +26 27 28 29 30 31 +""" + +result_2004_text = """\ + 2004 + + January February March +Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su + 1 2 3 4 1 1 2 3 4 5 6 7 + 5 6 7 8 9 10 11 2 3 4 5 6 7 8 8 9 10 11 12 13 14 +12 13 14 15 16 17 18 9 10 11 12 13 14 15 15 16 17 18 19 20 21 +19 20 21 22 23 24 25 16 17 18 19 20 21 22 22 23 24 25 26 27 28 +26 27 28 29 30 31 23 24 25 26 27 28 29 29 30 31 + + April May June +Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su + 1 2 3 4 1 2 1 2 3 4 5 6 + 5 6 7 8 9 10 11 3 4 5 6 7 8 9 7 8 9 10 11 12 13 +12 13 14 15 16 17 18 10 11 12 13 14 15 16 14 15 16 17 18 19 20 +19 20 21 22 23 24 25 17 18 19 20 21 22 23 21 22 23 24 25 26 27 +26 27 28 29 30 24 25 26 27 28 29 30 28 29 30 + 31 + + July August September +Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su + 1 2 3 4 1 1 2 3 4 5 + 5 6 7 8 9 10 11 2 3 4 5 6 7 8 6 7 8 9 10 11 12 +12 13 14 15 16 17 18 9 10 11 12 13 14 15 13 14 15 16 17 18 19 +19 20 21 22 23 24 25 16 17 18 19 20 21 22 20 21 22 23 24 25 26 +26 27 28 29 30 31 23 24 25 26 27 28 29 27 28 29 30 + 30 31 + + October November December +Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su + 1 2 3 1 2 3 4 5 6 7 1 2 3 4 5 + 4 5 6 7 8 9 10 8 9 10 11 12 13 14 6 7 8 9 10 11 12 +11 12 13 14 15 16 17 15 16 17 18 19 20 21 13 14 15 16 17 18 19 +18 19 20 21 22 23 24 22 23 24 25 26 27 28 20 21 22 23 24 25 26 +25 26 27 28 29 30 31 29 30 27 28 29 30 31 +""" + + +default_format = dict(year="year", month="month", encoding="ascii") + +result_2004_html = """\ + + + + + + +Calendar for 2004 + + + +
2004
+ + + + + + + +
January
MonTueWedThuFriSatSun
   1234
567891011
12131415161718
19202122232425
262728293031 
+		 + + + + + + + +
February
MonTueWedThuFriSatSun
      1
2345678
9101112131415
16171819202122
23242526272829
+		 + + + + + + + +
March
MonTueWedThuFriSatSun
1234567
891011121314
15161718192021
22232425262728
293031    
+
+ + + + + + + +
April
MonTueWedThuFriSatSun
   1234
567891011
12131415161718
19202122232425
2627282930  
+		 + + + + + + + + +
May
MonTueWedThuFriSatSun
     12
3456789
10111213141516
17181920212223
24252627282930
31      
+		 + + + + + + + +
June
MonTueWedThuFriSatSun
 123456
78910111213
14151617181920
21222324252627
282930    
+
+ + + + + + + +
July
MonTueWedThuFriSatSun
   1234
567891011
12131415161718
19202122232425
262728293031 
+		 + + + + + + + + +
August
MonTueWedThuFriSatSun
      1
2345678
9101112131415
16171819202122
23242526272829
3031     
+		 + + + + + + + +
September
MonTueWedThuFriSatSun
  12345
6789101112
13141516171819
20212223242526
27282930   
+
+ + + + + + + +
October
MonTueWedThuFriSatSun
    123
45678910
11121314151617
18192021222324
25262728293031
+		 + + + + + + + +
November
MonTueWedThuFriSatSun
1234567
891011121314
15161718192021
22232425262728
2930     
+		 + + + + + + + +
December
MonTueWedThuFriSatSun
  12345
6789101112
13141516171819
20212223242526
2728293031  
+
+ +""" + +result_2004_days = [ + [[[0, 0, 0, 1, 2, 3, 4], + [5, 6, 7, 8, 9, 10, 11], + [12, 13, 14, 15, 16, 17, 18], + [19, 20, 21, 22, 23, 24, 25], + [26, 27, 28, 29, 30, 31, 0]], + [[0, 0, 0, 0, 0, 0, 1], + [2, 3, 4, 5, 6, 7, 8], + [9, 10, 11, 12, 13, 14, 15], + [16, 17, 18, 19, 20, 21, 22], + [23, 24, 25, 26, 27, 28, 29]], + [[1, 2, 3, 4, 5, 6, 7], + [8, 9, 10, 11, 12, 13, 14], + [15, 16, 17, 18, 19, 20, 21], + [22, 23, 24, 25, 26, 27, 28], + [29, 30, 31, 0, 0, 0, 0]]], + [[[0, 0, 0, 1, 2, 3, 4], + [5, 6, 7, 8, 9, 10, 11], + [12, 13, 14, 15, 16, 17, 18], + [19, 20, 21, 22, 23, 24, 25], + [26, 27, 28, 29, 30, 0, 0]], + [[0, 0, 0, 0, 0, 1, 2], + [3, 4, 5, 6, 7, 8, 9], + [10, 11, 12, 13, 14, 15, 16], + [17, 18, 19, 20, 21, 22, 23], + [24, 25, 26, 27, 28, 29, 30], + [31, 0, 0, 0, 0, 0, 0]], + [[0, 1, 2, 3, 4, 5, 6], + [7, 8, 9, 10, 11, 12, 13], + [14, 15, 16, 17, 18, 19, 20], + [21, 22, 23, 24, 25, 26, 27], + [28, 29, 30, 0, 0, 0, 0]]], + [[[0, 0, 0, 1, 2, 3, 4], + [5, 6, 7, 8, 9, 10, 11], + [12, 13, 14, 15, 16, 17, 18], + [19, 20, 21, 22, 23, 24, 25], + [26, 27, 28, 29, 30, 31, 0]], + [[0, 0, 0, 0, 0, 0, 1], + [2, 3, 4, 5, 6, 7, 8], + [9, 10, 11, 12, 13, 14, 15], + [16, 17, 18, 19, 20, 21, 22], + [23, 24, 25, 26, 27, 28, 29], + [30, 31, 0, 0, 0, 0, 0]], + [[0, 0, 1, 2, 3, 4, 5], + [6, 7, 8, 9, 10, 11, 12], + [13, 14, 15, 16, 17, 18, 19], + [20, 21, 22, 23, 24, 25, 26], + [27, 28, 29, 30, 0, 0, 0]]], + [[[0, 0, 0, 0, 1, 2, 3], + [4, 5, 6, 7, 8, 9, 10], + [11, 12, 13, 14, 15, 16, 17], + [18, 19, 20, 21, 22, 23, 24], + [25, 26, 27, 28, 29, 30, 31]], + [[1, 2, 3, 4, 5, 6, 7], + [8, 9, 10, 11, 12, 13, 14], + [15, 16, 17, 18, 19, 20, 21], + [22, 23, 24, 25, 26, 27, 28], + [29, 30, 0, 0, 0, 0, 0]], + [[0, 0, 1, 2, 3, 4, 5], + [6, 7, 8, 9, 10, 11, 12], + [13, 14, 15, 16, 17, 18, 19], + [20, 21, 22, 23, 24, 25, 26], + [27, 28, 29, 30, 31, 0, 0]]] +] + +result_2004_dates = \ + [[['12/29/03 12/30/03 12/31/03 01/01/04 01/02/04 01/03/04 01/04/04', + '01/05/04 01/06/04 01/07/04 01/08/04 01/09/04 01/10/04 01/11/04', + '01/12/04 01/13/04 01/14/04 01/15/04 01/16/04 01/17/04 01/18/04', + '01/19/04 01/20/04 01/21/04 01/22/04 01/23/04 01/24/04 01/25/04', + '01/26/04 01/27/04 01/28/04 01/29/04 01/30/04 01/31/04 02/01/04'], + ['01/26/04 01/27/04 01/28/04 01/29/04 01/30/04 01/31/04 02/01/04', + '02/02/04 02/03/04 02/04/04 02/05/04 02/06/04 02/07/04 02/08/04', + '02/09/04 02/10/04 02/11/04 02/12/04 02/13/04 02/14/04 02/15/04', + '02/16/04 02/17/04 02/18/04 02/19/04 02/20/04 02/21/04 02/22/04', + '02/23/04 02/24/04 02/25/04 02/26/04 02/27/04 02/28/04 02/29/04'], + ['03/01/04 03/02/04 03/03/04 03/04/04 03/05/04 03/06/04 03/07/04', + '03/08/04 03/09/04 03/10/04 03/11/04 03/12/04 03/13/04 03/14/04', + '03/15/04 03/16/04 03/17/04 03/18/04 03/19/04 03/20/04 03/21/04', + '03/22/04 03/23/04 03/24/04 03/25/04 03/26/04 03/27/04 03/28/04', + '03/29/04 03/30/04 03/31/04 04/01/04 04/02/04 04/03/04 04/04/04']], + [['03/29/04 03/30/04 03/31/04 04/01/04 04/02/04 04/03/04 04/04/04', + '04/05/04 04/06/04 04/07/04 04/08/04 04/09/04 04/10/04 04/11/04', + '04/12/04 04/13/04 04/14/04 04/15/04 04/16/04 04/17/04 04/18/04', + '04/19/04 04/20/04 04/21/04 04/22/04 04/23/04 04/24/04 04/25/04', + '04/26/04 04/27/04 04/28/04 04/29/04 04/30/04 05/01/04 05/02/04'], + ['04/26/04 04/27/04 04/28/04 04/29/04 04/30/04 05/01/04 05/02/04', + '05/03/04 05/04/04 05/05/04 05/06/04 05/07/04 05/08/04 05/09/04', + '05/10/04 05/11/04 05/12/04 05/13/04 05/14/04 05/15/04 05/16/04', + '05/17/04 05/18/04 05/19/04 05/20/04 05/21/04 05/22/04 05/23/04', + '05/24/04 05/25/04 05/26/04 05/27/04 05/28/04 05/29/04 05/30/04', + '05/31/04 06/01/04 06/02/04 06/03/04 06/04/04 06/05/04 06/06/04'], + ['05/31/04 06/01/04 06/02/04 06/03/04 06/04/04 06/05/04 06/06/04', + '06/07/04 06/08/04 06/09/04 06/10/04 06/11/04 06/12/04 06/13/04', + '06/14/04 06/15/04 06/16/04 06/17/04 06/18/04 06/19/04 06/20/04', + '06/21/04 06/22/04 06/23/04 06/24/04 06/25/04 06/26/04 06/27/04', + '06/28/04 06/29/04 06/30/04 07/01/04 07/02/04 07/03/04 07/04/04']], + [['06/28/04 06/29/04 06/30/04 07/01/04 07/02/04 07/03/04 07/04/04', + '07/05/04 07/06/04 07/07/04 07/08/04 07/09/04 07/10/04 07/11/04', + '07/12/04 07/13/04 07/14/04 07/15/04 07/16/04 07/17/04 07/18/04', + '07/19/04 07/20/04 07/21/04 07/22/04 07/23/04 07/24/04 07/25/04', + '07/26/04 07/27/04 07/28/04 07/29/04 07/30/04 07/31/04 08/01/04'], + ['07/26/04 07/27/04 07/28/04 07/29/04 07/30/04 07/31/04 08/01/04', + '08/02/04 08/03/04 08/04/04 08/05/04 08/06/04 08/07/04 08/08/04', + '08/09/04 08/10/04 08/11/04 08/12/04 08/13/04 08/14/04 08/15/04', + '08/16/04 08/17/04 08/18/04 08/19/04 08/20/04 08/21/04 08/22/04', + '08/23/04 08/24/04 08/25/04 08/26/04 08/27/04 08/28/04 08/29/04', + '08/30/04 08/31/04 09/01/04 09/02/04 09/03/04 09/04/04 09/05/04'], + ['08/30/04 08/31/04 09/01/04 09/02/04 09/03/04 09/04/04 09/05/04', + '09/06/04 09/07/04 09/08/04 09/09/04 09/10/04 09/11/04 09/12/04', + '09/13/04 09/14/04 09/15/04 09/16/04 09/17/04 09/18/04 09/19/04', + '09/20/04 09/21/04 09/22/04 09/23/04 09/24/04 09/25/04 09/26/04', + '09/27/04 09/28/04 09/29/04 09/30/04 10/01/04 10/02/04 10/03/04']], + [['09/27/04 09/28/04 09/29/04 09/30/04 10/01/04 10/02/04 10/03/04', + '10/04/04 10/05/04 10/06/04 10/07/04 10/08/04 10/09/04 10/10/04', + '10/11/04 10/12/04 10/13/04 10/14/04 10/15/04 10/16/04 10/17/04', + '10/18/04 10/19/04 10/20/04 10/21/04 10/22/04 10/23/04 10/24/04', + '10/25/04 10/26/04 10/27/04 10/28/04 10/29/04 10/30/04 10/31/04'], + ['11/01/04 11/02/04 11/03/04 11/04/04 11/05/04 11/06/04 11/07/04', + '11/08/04 11/09/04 11/10/04 11/11/04 11/12/04 11/13/04 11/14/04', + '11/15/04 11/16/04 11/17/04 11/18/04 11/19/04 11/20/04 11/21/04', + '11/22/04 11/23/04 11/24/04 11/25/04 11/26/04 11/27/04 11/28/04', + '11/29/04 11/30/04 12/01/04 12/02/04 12/03/04 12/04/04 12/05/04'], + ['11/29/04 11/30/04 12/01/04 12/02/04 12/03/04 12/04/04 12/05/04', + '12/06/04 12/07/04 12/08/04 12/09/04 12/10/04 12/11/04 12/12/04', + '12/13/04 12/14/04 12/15/04 12/16/04 12/17/04 12/18/04 12/19/04', + '12/20/04 12/21/04 12/22/04 12/23/04 12/24/04 12/25/04 12/26/04', + '12/27/04 12/28/04 12/29/04 12/30/04 12/31/04 01/01/05 01/02/05']]] + + +class OutputTestCase(unittest.TestCase): + def normalize_calendar(self, s): + # Filters out locale dependent strings + def neitherspacenordigit(c): + return not c.isspace() and not c.isdigit() + + lines = [] + for line in s.splitlines(keepends=False): + # Drop texts, as they are locale dependent + if line and not filter(neitherspacenordigit, line): + lines.append(line) + return lines + + def check_htmlcalendar_encoding(self, req, res): + cal = calendar.HTMLCalendar() + format_ = default_format.copy() + format_["encoding"] = req or 'utf-8' + output = cal.formatyearpage(2004, encoding=req) + self.assertEqual( + output, + result_2004_html.format(**format_).encode(res) + ) + + def test_output(self): + self.assertEqual( + self.normalize_calendar(calendar.calendar(2004)), + self.normalize_calendar(result_2004_text) + ) + self.assertEqual( + self.normalize_calendar(calendar.calendar(0)), + self.normalize_calendar(result_0_text) + ) + + def test_output_textcalendar(self): + self.assertEqual( + calendar.TextCalendar().formatyear(2004), + result_2004_text + ) + self.assertEqual( + calendar.TextCalendar().formatyear(0), + result_0_text + ) + + def test_output_htmlcalendar_encoding_ascii(self): + self.check_htmlcalendar_encoding('ascii', 'ascii') + + def test_output_htmlcalendar_encoding_utf8(self): + self.check_htmlcalendar_encoding('utf-8', 'utf-8') + + def test_output_htmlcalendar_encoding_default(self): + self.check_htmlcalendar_encoding(None, sys.getdefaultencoding()) + + def test_yeardatescalendar(self): + def shrink(cal): + return [[[' '.join('{:02d}/{:02d}/{}'.format( + d.month, d.day, str(d.year)[-2:]) for d in z) + for z in y] for y in x] for x in cal] + self.assertEqual( + shrink(calendar.Calendar().yeardatescalendar(2004)), + result_2004_dates + ) + + def test_yeardayscalendar(self): + self.assertEqual( + calendar.Calendar().yeardayscalendar(2004), + result_2004_days + ) + + def test_formatweekheader_short(self): + self.assertEqual( + calendar.TextCalendar().formatweekheader(2), + 'Mo Tu We Th Fr Sa Su' + ) + + def test_formatweekheader_long(self): + self.assertEqual( + calendar.TextCalendar().formatweekheader(9), + ' Monday Tuesday Wednesday Thursday ' + ' Friday Saturday Sunday ' + ) + + def test_formatmonth(self): + self.assertEqual( + calendar.TextCalendar().formatmonth(2004, 1), + result_2004_01_text + ) + self.assertEqual( + calendar.TextCalendar().formatmonth(0, 2), + result_0_02_text + ) + + def test_formatmonthname_with_year(self): + self.assertEqual( + calendar.HTMLCalendar().formatmonthname(2004, 1, withyear=True), + 'January 2004' + ) + + def test_formatmonthname_without_year(self): + self.assertEqual( + calendar.HTMLCalendar().formatmonthname(2004, 1, withyear=False), + 'January' + ) + + def test_prweek(self): + with support.captured_stdout() as out: + week = [(1,0), (2,1), (3,2), (4,3), (5,4), (6,5), (7,6)] + calendar.TextCalendar().prweek(week, 1) + self.assertEqual(out.getvalue(), " 1 2 3 4 5 6 7") + + def test_prmonth(self): + with support.captured_stdout() as out: + calendar.TextCalendar().prmonth(2004, 1) + self.assertEqual(out.getvalue(), result_2004_01_text) + + def test_pryear(self): + with support.captured_stdout() as out: + calendar.TextCalendar().pryear(2004) + self.assertEqual(out.getvalue(), result_2004_text) + + def test_format(self): + with support.captured_stdout() as out: + calendar.format(["1", "2", "3"], colwidth=3, spacing=1) + self.assertEqual(out.getvalue().strip(), "1 2 3") + +class CalendarTestCase(unittest.TestCase): + def test_isleap(self): + # Make sure that the return is right for a few years, and + # ensure that the return values are 1 or 0, not just true or + # false (see SF bug #485794). Specific additional tests may + # be appropriate; this tests a single "cycle". + self.assertEqual(calendar.isleap(2000), 1) + self.assertEqual(calendar.isleap(2001), 0) + self.assertEqual(calendar.isleap(2002), 0) + self.assertEqual(calendar.isleap(2003), 0) + + def test_setfirstweekday(self): + self.assertRaises(TypeError, calendar.setfirstweekday, 'flabber') + self.assertRaises(ValueError, calendar.setfirstweekday, -1) + self.assertRaises(ValueError, calendar.setfirstweekday, 200) + orig = calendar.firstweekday() + calendar.setfirstweekday(calendar.SUNDAY) + self.assertEqual(calendar.firstweekday(), calendar.SUNDAY) + calendar.setfirstweekday(calendar.MONDAY) + self.assertEqual(calendar.firstweekday(), calendar.MONDAY) + calendar.setfirstweekday(orig) + + def test_illegal_weekday_reported(self): + with self.assertRaisesRegex(calendar.IllegalWeekdayError, '123'): + calendar.setfirstweekday(123) + + def test_enumerate_weekdays(self): + self.assertRaises(IndexError, calendar.day_abbr.__getitem__, -10) + self.assertRaises(IndexError, calendar.day_name.__getitem__, 10) + self.assertEqual(len([d for d in calendar.day_abbr]), 7) + + def test_days(self): + for attr in "day_name", "day_abbr": + value = getattr(calendar, attr) + self.assertEqual(len(value), 7) + self.assertEqual(len(value[:]), 7) + # ensure they're all unique + self.assertEqual(len(set(value)), 7) + # verify it "acts like a sequence" in two forms of iteration + self.assertEqual(value[::-1], list(reversed(value))) + + def test_months(self): + for attr in "month_name", "month_abbr": + value = getattr(calendar, attr) + self.assertEqual(len(value), 13) + self.assertEqual(len(value[:]), 13) + self.assertEqual(value[0], "") + # ensure they're all unique + self.assertEqual(len(set(value)), 13) + # verify it "acts like a sequence" in two forms of iteration + self.assertEqual(value[::-1], list(reversed(value))) + + def test_locale_calendars(self): + # ensure that Locale{Text,HTML}Calendar resets the locale properly + # (it is still not thread-safe though) + old_october = calendar.TextCalendar().formatmonthname(2010, 10, 10) + try: + cal = calendar.LocaleTextCalendar(locale='') + local_weekday = cal.formatweekday(1, 10) + local_month = cal.formatmonthname(2010, 10, 10) + except locale.Error: + # cannot set the system default locale -- skip rest of test + raise unittest.SkipTest('cannot set the system default locale') + self.assertIsInstance(local_weekday, str) + self.assertIsInstance(local_month, str) + self.assertEqual(len(local_weekday), 10) + self.assertGreaterEqual(len(local_month), 10) + cal = calendar.LocaleHTMLCalendar(locale='') + local_weekday = cal.formatweekday(1) + local_month = cal.formatmonthname(2010, 10) + self.assertIsInstance(local_weekday, str) + self.assertIsInstance(local_month, str) + new_october = calendar.TextCalendar().formatmonthname(2010, 10, 10) + self.assertEqual(old_october, new_october) + + def test_itermonthdays3(self): + # ensure itermonthdays3 doesn't overflow after datetime.MAXYEAR + list(calendar.Calendar().itermonthdays3(datetime.MAXYEAR, 12)) + + def test_itermonthdays4(self): + cal = calendar.Calendar(firstweekday=3) + days = list(cal.itermonthdays4(2001, 2)) + self.assertEqual(days[0], (2001, 2, 1, 3)) + self.assertEqual(days[-1], (2001, 2, 28, 2)) + + def test_itermonthdays(self): + for firstweekday in range(7): + cal = calendar.Calendar(firstweekday) + # Test the extremes, see #28253 and #26650 + for y, m in [(1, 1), (9999, 12)]: + days = list(cal.itermonthdays(y, m)) + self.assertIn(len(days), (35, 42)) + # Test a short month + cal = calendar.Calendar(firstweekday=3) + days = list(cal.itermonthdays(2001, 2)) + self.assertEqual(days, list(range(1, 29))) + + def test_itermonthdays2(self): + for firstweekday in range(7): + cal = calendar.Calendar(firstweekday) + # Test the extremes, see #28253 and #26650 + for y, m in [(1, 1), (9999, 12)]: + days = list(cal.itermonthdays2(y, m)) + self.assertEqual(days[0][1], firstweekday) + self.assertEqual(days[-1][1], (firstweekday - 1) % 7) + + +class MonthCalendarTestCase(unittest.TestCase): + def setUp(self): + self.oldfirstweekday = calendar.firstweekday() + calendar.setfirstweekday(self.firstweekday) + + def tearDown(self): + calendar.setfirstweekday(self.oldfirstweekday) + + def check_weeks(self, year, month, weeks): + cal = calendar.monthcalendar(year, month) + self.assertEqual(len(cal), len(weeks)) + for i in range(len(weeks)): + self.assertEqual(weeks[i], sum(day != 0 for day in cal[i])) + + +class MondayTestCase(MonthCalendarTestCase): + firstweekday = calendar.MONDAY + + def test_february(self): + # A 28-day february starting on monday (7+7+7+7 days) + self.check_weeks(1999, 2, (7, 7, 7, 7)) + + # A 28-day february starting on tuesday (6+7+7+7+1 days) + self.check_weeks(2005, 2, (6, 7, 7, 7, 1)) + + # A 28-day february starting on sunday (1+7+7+7+6 days) + self.check_weeks(1987, 2, (1, 7, 7, 7, 6)) + + # A 29-day february starting on monday (7+7+7+7+1 days) + self.check_weeks(1988, 2, (7, 7, 7, 7, 1)) + + # A 29-day february starting on tuesday (6+7+7+7+2 days) + self.check_weeks(1972, 2, (6, 7, 7, 7, 2)) + + # A 29-day february starting on sunday (1+7+7+7+7 days) + self.check_weeks(2004, 2, (1, 7, 7, 7, 7)) + + def test_april(self): + # A 30-day april starting on monday (7+7+7+7+2 days) + self.check_weeks(1935, 4, (7, 7, 7, 7, 2)) + + # A 30-day april starting on tuesday (6+7+7+7+3 days) + self.check_weeks(1975, 4, (6, 7, 7, 7, 3)) + + # A 30-day april starting on sunday (1+7+7+7+7+1 days) + self.check_weeks(1945, 4, (1, 7, 7, 7, 7, 1)) + + # A 30-day april starting on saturday (2+7+7+7+7 days) + self.check_weeks(1995, 4, (2, 7, 7, 7, 7)) + + # A 30-day april starting on friday (3+7+7+7+6 days) + self.check_weeks(1994, 4, (3, 7, 7, 7, 6)) + + def test_december(self): + # A 31-day december starting on monday (7+7+7+7+3 days) + self.check_weeks(1980, 12, (7, 7, 7, 7, 3)) + + # A 31-day december starting on tuesday (6+7+7+7+4 days) + self.check_weeks(1987, 12, (6, 7, 7, 7, 4)) + + # A 31-day december starting on sunday (1+7+7+7+7+2 days) + self.check_weeks(1968, 12, (1, 7, 7, 7, 7, 2)) + + # A 31-day december starting on thursday (4+7+7+7+6 days) + self.check_weeks(1988, 12, (4, 7, 7, 7, 6)) + + # A 31-day december starting on friday (3+7+7+7+7 days) + self.check_weeks(2017, 12, (3, 7, 7, 7, 7)) + + # A 31-day december starting on saturday (2+7+7+7+7+1 days) + self.check_weeks(2068, 12, (2, 7, 7, 7, 7, 1)) + + +class SundayTestCase(MonthCalendarTestCase): + firstweekday = calendar.SUNDAY + + def test_february(self): + # A 28-day february starting on sunday (7+7+7+7 days) + self.check_weeks(2009, 2, (7, 7, 7, 7)) + + # A 28-day february starting on monday (6+7+7+7+1 days) + self.check_weeks(1999, 2, (6, 7, 7, 7, 1)) + + # A 28-day february starting on saturday (1+7+7+7+6 days) + self.check_weeks(1997, 2, (1, 7, 7, 7, 6)) + + # A 29-day february starting on sunday (7+7+7+7+1 days) + self.check_weeks(2004, 2, (7, 7, 7, 7, 1)) + + # A 29-day february starting on monday (6+7+7+7+2 days) + self.check_weeks(1960, 2, (6, 7, 7, 7, 2)) + + # A 29-day february starting on saturday (1+7+7+7+7 days) + self.check_weeks(1964, 2, (1, 7, 7, 7, 7)) + + def test_april(self): + # A 30-day april starting on sunday (7+7+7+7+2 days) + self.check_weeks(1923, 4, (7, 7, 7, 7, 2)) + + # A 30-day april starting on monday (6+7+7+7+3 days) + self.check_weeks(1918, 4, (6, 7, 7, 7, 3)) + + # A 30-day april starting on saturday (1+7+7+7+7+1 days) + self.check_weeks(1950, 4, (1, 7, 7, 7, 7, 1)) + + # A 30-day april starting on friday (2+7+7+7+7 days) + self.check_weeks(1960, 4, (2, 7, 7, 7, 7)) + + # A 30-day april starting on thursday (3+7+7+7+6 days) + self.check_weeks(1909, 4, (3, 7, 7, 7, 6)) + + def test_december(self): + # A 31-day december starting on sunday (7+7+7+7+3 days) + self.check_weeks(2080, 12, (7, 7, 7, 7, 3)) + + # A 31-day december starting on monday (6+7+7+7+4 days) + self.check_weeks(1941, 12, (6, 7, 7, 7, 4)) + + # A 31-day december starting on saturday (1+7+7+7+7+2 days) + self.check_weeks(1923, 12, (1, 7, 7, 7, 7, 2)) + + # A 31-day december starting on wednesday (4+7+7+7+6 days) + self.check_weeks(1948, 12, (4, 7, 7, 7, 6)) + + # A 31-day december starting on thursday (3+7+7+7+7 days) + self.check_weeks(1927, 12, (3, 7, 7, 7, 7)) + + # A 31-day december starting on friday (2+7+7+7+7+1 days) + self.check_weeks(1995, 12, (2, 7, 7, 7, 7, 1)) + +class TimegmTestCase(unittest.TestCase): + TIMESTAMPS = [0, 10, 100, 1000, 10000, 100000, 1000000, + 1234567890, 1262304000, 1275785153,] + def test_timegm(self): + for secs in self.TIMESTAMPS: + tuple = time.gmtime(secs) + self.assertEqual(secs, calendar.timegm(tuple)) + +class MonthRangeTestCase(unittest.TestCase): + def test_january(self): + # Tests valid lower boundary case. + self.assertEqual(calendar.monthrange(2004,1), (3,31)) + + def test_february_leap(self): + # Tests February during leap year. + self.assertEqual(calendar.monthrange(2004,2), (6,29)) + + def test_february_nonleap(self): + # Tests February in non-leap year. + self.assertEqual(calendar.monthrange(2010,2), (0,28)) + + def test_december(self): + # Tests valid upper boundary case. + self.assertEqual(calendar.monthrange(2004,12), (2,31)) + + def test_zeroth_month(self): + # Tests low invalid boundary case. + with self.assertRaises(calendar.IllegalMonthError): + calendar.monthrange(2004, 0) + + def test_thirteenth_month(self): + # Tests high invalid boundary case. + with self.assertRaises(calendar.IllegalMonthError): + calendar.monthrange(2004, 13) + + def test_illegal_month_reported(self): + with self.assertRaisesRegex(calendar.IllegalMonthError, '65'): + calendar.monthrange(2004, 65) + +class LeapdaysTestCase(unittest.TestCase): + def test_no_range(self): + # test when no range i.e. two identical years as args + self.assertEqual(calendar.leapdays(2010,2010), 0) + + def test_no_leapdays(self): + # test when no leap years in range + self.assertEqual(calendar.leapdays(2010,2011), 0) + + def test_no_leapdays_upper_boundary(self): + # test no leap years in range, when upper boundary is a leap year + self.assertEqual(calendar.leapdays(2010,2012), 0) + + def test_one_leapday_lower_boundary(self): + # test when one leap year in range, lower boundary is leap year + self.assertEqual(calendar.leapdays(2012,2013), 1) + + def test_several_leapyears_in_range(self): + self.assertEqual(calendar.leapdays(1997,2020), 5) + + +def conv(s): + return s.replace('\n', os.linesep).encode() + +class CommandLineTestCase(unittest.TestCase): + def run_ok(self, *args): + return assert_python_ok('-m', 'calendar', *args)[1] + + def assertFailure(self, *args): + rc, stdout, stderr = assert_python_failure('-m', 'calendar', *args) + self.assertIn(b'usage:', stderr) + self.assertEqual(rc, 2) + + def test_help(self): + stdout = self.run_ok('-h') + self.assertIn(b'usage:', stdout) + self.assertIn(b'calendar.py', stdout) + self.assertIn(b'--help', stdout) + + def test_illegal_arguments(self): + self.assertFailure('-z') + self.assertFailure('spam') + self.assertFailure('2004', 'spam') + self.assertFailure('-t', 'html', '2004', '1') + + def test_output_current_year(self): + stdout = self.run_ok() + year = datetime.datetime.now().year + self.assertIn((' %s' % year).encode(), stdout) + self.assertIn(b'January', stdout) + self.assertIn(b'Mo Tu We Th Fr Sa Su', stdout) + + def test_output_year(self): + stdout = self.run_ok('2004') + self.assertEqual(stdout, conv(result_2004_text)) + + def test_output_month(self): + stdout = self.run_ok('2004', '1') + self.assertEqual(stdout, conv(result_2004_01_text)) + + def test_option_encoding(self): + self.assertFailure('-e') + self.assertFailure('--encoding') + stdout = self.run_ok('--encoding', 'utf-16-le', '2004') + self.assertEqual(stdout, result_2004_text.encode('utf-16-le')) + + def test_option_locale(self): + self.assertFailure('-L') + self.assertFailure('--locale') + self.assertFailure('-L', 'en') + lang, enc = locale.getdefaultlocale() + lang = lang or 'C' + enc = enc or 'UTF-8' + try: + oldlocale = locale.getlocale(locale.LC_TIME) + try: + locale.setlocale(locale.LC_TIME, (lang, enc)) + finally: + locale.setlocale(locale.LC_TIME, oldlocale) + except (locale.Error, ValueError): + self.skipTest('cannot set the system default locale') + stdout = self.run_ok('--locale', lang, '--encoding', enc, '2004') + self.assertIn('2004'.encode(enc), stdout) + + def test_option_width(self): + self.assertFailure('-w') + self.assertFailure('--width') + self.assertFailure('-w', 'spam') + stdout = self.run_ok('--width', '3', '2004') + self.assertIn(b'Mon Tue Wed Thu Fri Sat Sun', stdout) + + def test_option_lines(self): + self.assertFailure('-l') + self.assertFailure('--lines') + self.assertFailure('-l', 'spam') + stdout = self.run_ok('--lines', '2', '2004') + self.assertIn(conv('December\n\nMo Tu We'), stdout) + + def test_option_spacing(self): + self.assertFailure('-s') + self.assertFailure('--spacing') + self.assertFailure('-s', 'spam') + stdout = self.run_ok('--spacing', '8', '2004') + self.assertIn(b'Su Mo', stdout) + + def test_option_months(self): + self.assertFailure('-m') + self.assertFailure('--month') + self.assertFailure('-m', 'spam') + stdout = self.run_ok('--months', '1', '2004') + self.assertIn(conv('\nMo Tu We Th Fr Sa Su\n'), stdout) + + def test_option_type(self): + self.assertFailure('-t') + self.assertFailure('--type') + self.assertFailure('-t', 'spam') + stdout = self.run_ok('--type', 'text', '2004') + self.assertEqual(stdout, conv(result_2004_text)) + stdout = self.run_ok('--type', 'html', '2004') + self.assertEqual(stdout[:6], b'Calendar for 2004', stdout) + + def test_html_output_current_year(self): + stdout = self.run_ok('--type', 'html') + year = datetime.datetime.now().year + self.assertIn(('Calendar for %s' % year).encode(), + stdout) + self.assertIn(b'January', + stdout) + + def test_html_output_year_encoding(self): + stdout = self.run_ok('-t', 'html', '--encoding', 'ascii', '2004') + self.assertEqual(stdout, + result_2004_html.format(**default_format).encode('ascii')) + + def test_html_output_year_css(self): + self.assertFailure('-t', 'html', '-c') + self.assertFailure('-t', 'html', '--css') + stdout = self.run_ok('-t', 'html', '--css', 'custom.css', '2004') + self.assertIn(b'', stdout) + + +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = {'mdays', 'January', 'February', 'EPOCH', + 'MONDAY', 'TUESDAY', 'WEDNESDAY', 'THURSDAY', 'FRIDAY', + 'SATURDAY', 'SUNDAY', 'different_locale', 'c', + 'prweek', 'week', 'format', 'formatstring', 'main', + 'monthlen', 'prevmonth', 'nextmonth'} + support.check__all__(self, calendar, blacklist=blacklist) + + +class TestSubClassingCase(unittest.TestCase): + + def setUp(self): + + class CustomHTMLCal(calendar.HTMLCalendar): + cssclasses = [style + " text-nowrap" for style in + calendar.HTMLCalendar.cssclasses] + cssclasses_weekday_head = ["red", "blue", "green", "lilac", + "yellow", "orange", "pink"] + cssclass_month_head = "text-center month-head" + cssclass_month = "text-center month" + cssclass_year = "text-italic " + cssclass_year_head = "lead " + + self.cal = CustomHTMLCal() + + def test_formatmonthname(self): + self.assertIn('class="text-center month-head"', + self.cal.formatmonthname(2017, 5)) + + def test_formatmonth(self): + self.assertIn('class="text-center month"', + self.cal.formatmonth(2017, 5)) + + def test_formatweek(self): + weeks = self.cal.monthdays2calendar(2017, 5) + self.assertIn('class="wed text-nowrap"', self.cal.formatweek(weeks[0])) + + def test_formatweek_head(self): + header = self.cal.formatweekheader() + for color in self.cal.cssclasses_weekday_head: + self.assertIn('' % color, header) + + def test_format_year(self): + self.assertIn( + ('' % + self.cal.cssclass_year), self.cal.formatyear(2017)) + + def test_format_year_head(self): + self.assertIn('' % ( + 3, self.cal.cssclass_year_head, 2017), self.cal.formatyear(2017)) + +if __name__ == "__main__": + unittest.main() From 808b1cdc9d99121860f5eef474df72dfb0d0ac30 Mon Sep 17 00:00:00 2001 From: stu Date: Sun, 3 Apr 2022 22:07:32 +0800 Subject: [PATCH 038/137] adjust tests for skulpt --- test/unit3/test_calendar.py | 181 ++++++++++-------------------------- 1 file changed, 49 insertions(+), 132 deletions(-) diff --git a/test/unit3/test_calendar.py b/test/unit3/test_calendar.py index 6241d114d3..ccf34683f4 100644 --- a/test/unit3/test_calendar.py +++ b/test/unit3/test_calendar.py @@ -1,13 +1,34 @@ import calendar import unittest -from test import support -from test.support.script_helper import assert_python_ok, assert_python_failure +# from test import support +# from test.support.script_helper import assert_python_ok, assert_python_failure import time -import locale +# import locale import sys import datetime -import os +# import os + +class captured_stdout(): + """Return a context manager used by captured_stdout/stdin/stderr + that temporarily replaces the sys stream *stream_name* with a StringIO.""" + def __init__(self): + self.captured = "" + + def write(self, x): + self.captured += x + + def getvalue(self): + return self.captured + + def __enter__(self): + self.captured = "" + sys.stdout = self + return self + + def __exit__(self, *args): + sys.stdout = sys.__stdout__ + # From https://en.wikipedia.org/wiki/Leap_year_starting_on_Saturday result_0_02_text = """\ @@ -469,23 +490,23 @@ def test_formatmonthname_without_year(self): ) def test_prweek(self): - with support.captured_stdout() as out: + with captured_stdout() as out: week = [(1,0), (2,1), (3,2), (4,3), (5,4), (6,5), (7,6)] calendar.TextCalendar().prweek(week, 1) self.assertEqual(out.getvalue(), " 1 2 3 4 5 6 7") def test_prmonth(self): - with support.captured_stdout() as out: + with captured_stdout() as out: calendar.TextCalendar().prmonth(2004, 1) self.assertEqual(out.getvalue(), result_2004_01_text) def test_pryear(self): - with support.captured_stdout() as out: + with captured_stdout() as out: calendar.TextCalendar().pryear(2004) self.assertEqual(out.getvalue(), result_2004_text) def test_format(self): - with support.captured_stdout() as out: + with captured_stdout() as out: calendar.format(["1", "2", "3"], colwidth=3, spacing=1) self.assertEqual(out.getvalue().strip(), "1 2 3") @@ -512,8 +533,9 @@ def test_setfirstweekday(self): calendar.setfirstweekday(orig) def test_illegal_weekday_reported(self): - with self.assertRaisesRegex(calendar.IllegalWeekdayError, '123'): + with self.assertRaises(calendar.IllegalWeekdayError) as e: calendar.setfirstweekday(123) + self.assertIn("123", str(e.exception)) def test_enumerate_weekdays(self): self.assertRaises(IndexError, calendar.day_abbr.__getitem__, -10) @@ -549,7 +571,7 @@ def test_locale_calendars(self): cal = calendar.LocaleTextCalendar(locale='') local_weekday = cal.formatweekday(1, 10) local_month = cal.formatmonthname(2010, 10, 10) - except locale.Error: + except Exception: # cannot set the system default locale -- skip rest of test raise unittest.SkipTest('cannot set the system default locale') self.assertIsInstance(local_weekday, str) @@ -762,8 +784,9 @@ def test_thirteenth_month(self): calendar.monthrange(2004, 13) def test_illegal_month_reported(self): - with self.assertRaisesRegex(calendar.IllegalMonthError, '65'): + with self.assertRaises(calendar.IllegalMonthError) as e: calendar.monthrange(2004, 65) + self.assertIn("65", str(e.exception)) class LeapdaysTestCase(unittest.TestCase): def test_no_range(self): @@ -786,126 +809,6 @@ def test_several_leapyears_in_range(self): self.assertEqual(calendar.leapdays(1997,2020), 5) -def conv(s): - return s.replace('\n', os.linesep).encode() - -class CommandLineTestCase(unittest.TestCase): - def run_ok(self, *args): - return assert_python_ok('-m', 'calendar', *args)[1] - - def assertFailure(self, *args): - rc, stdout, stderr = assert_python_failure('-m', 'calendar', *args) - self.assertIn(b'usage:', stderr) - self.assertEqual(rc, 2) - - def test_help(self): - stdout = self.run_ok('-h') - self.assertIn(b'usage:', stdout) - self.assertIn(b'calendar.py', stdout) - self.assertIn(b'--help', stdout) - - def test_illegal_arguments(self): - self.assertFailure('-z') - self.assertFailure('spam') - self.assertFailure('2004', 'spam') - self.assertFailure('-t', 'html', '2004', '1') - - def test_output_current_year(self): - stdout = self.run_ok() - year = datetime.datetime.now().year - self.assertIn((' %s' % year).encode(), stdout) - self.assertIn(b'January', stdout) - self.assertIn(b'Mo Tu We Th Fr Sa Su', stdout) - - def test_output_year(self): - stdout = self.run_ok('2004') - self.assertEqual(stdout, conv(result_2004_text)) - - def test_output_month(self): - stdout = self.run_ok('2004', '1') - self.assertEqual(stdout, conv(result_2004_01_text)) - - def test_option_encoding(self): - self.assertFailure('-e') - self.assertFailure('--encoding') - stdout = self.run_ok('--encoding', 'utf-16-le', '2004') - self.assertEqual(stdout, result_2004_text.encode('utf-16-le')) - - def test_option_locale(self): - self.assertFailure('-L') - self.assertFailure('--locale') - self.assertFailure('-L', 'en') - lang, enc = locale.getdefaultlocale() - lang = lang or 'C' - enc = enc or 'UTF-8' - try: - oldlocale = locale.getlocale(locale.LC_TIME) - try: - locale.setlocale(locale.LC_TIME, (lang, enc)) - finally: - locale.setlocale(locale.LC_TIME, oldlocale) - except (locale.Error, ValueError): - self.skipTest('cannot set the system default locale') - stdout = self.run_ok('--locale', lang, '--encoding', enc, '2004') - self.assertIn('2004'.encode(enc), stdout) - - def test_option_width(self): - self.assertFailure('-w') - self.assertFailure('--width') - self.assertFailure('-w', 'spam') - stdout = self.run_ok('--width', '3', '2004') - self.assertIn(b'Mon Tue Wed Thu Fri Sat Sun', stdout) - - def test_option_lines(self): - self.assertFailure('-l') - self.assertFailure('--lines') - self.assertFailure('-l', 'spam') - stdout = self.run_ok('--lines', '2', '2004') - self.assertIn(conv('December\n\nMo Tu We'), stdout) - - def test_option_spacing(self): - self.assertFailure('-s') - self.assertFailure('--spacing') - self.assertFailure('-s', 'spam') - stdout = self.run_ok('--spacing', '8', '2004') - self.assertIn(b'Su Mo', stdout) - - def test_option_months(self): - self.assertFailure('-m') - self.assertFailure('--month') - self.assertFailure('-m', 'spam') - stdout = self.run_ok('--months', '1', '2004') - self.assertIn(conv('\nMo Tu We Th Fr Sa Su\n'), stdout) - - def test_option_type(self): - self.assertFailure('-t') - self.assertFailure('--type') - self.assertFailure('-t', 'spam') - stdout = self.run_ok('--type', 'text', '2004') - self.assertEqual(stdout, conv(result_2004_text)) - stdout = self.run_ok('--type', 'html', '2004') - self.assertEqual(stdout[:6], b'Calendar for 2004', stdout) - - def test_html_output_current_year(self): - stdout = self.run_ok('--type', 'html') - year = datetime.datetime.now().year - self.assertIn(('Calendar for %s' % year).encode(), - stdout) - self.assertIn(b'', - stdout) - - def test_html_output_year_encoding(self): - stdout = self.run_ok('-t', 'html', '--encoding', 'ascii', '2004') - self.assertEqual(stdout, - result_2004_html.format(**default_format).encode('ascii')) - - def test_html_output_year_css(self): - self.assertFailure('-t', 'html', '-c') - self.assertFailure('-t', 'html', '--css') - stdout = self.run_ok('-t', 'html', '--css', 'custom.css', '2004') - self.assertIn(b'', stdout) class MiscTestCase(unittest.TestCase): @@ -915,7 +818,21 @@ def test__all__(self): 'SATURDAY', 'SUNDAY', 'different_locale', 'c', 'prweek', 'week', 'format', 'formatstring', 'main', 'monthlen', 'prevmonth', 'nextmonth'} - support.check__all__(self, calendar, blacklist=blacklist) + # __all__ = set(calendar.__all__) + name_of_module = ("calendar", ) + expected = set() + ModuleType = type(calendar) + + for name in dir(calendar): + if name.startswith('_') or name in blacklist: + continue + obj = getattr(calendar, name) + if (getattr(obj, '__module__', None) in name_of_module or + (not hasattr(obj, '__module__') and + not isinstance(obj, ModuleType))): + expected.add(name) + self.assertEqual(set(calendar.__all__), expected) + class TestSubClassingCase(unittest.TestCase): From 53a76bc97d18bed6b6fb96d1856feca83ddcfb07 Mon Sep 17 00:00:00 2001 From: stu Date: Sun, 3 Apr 2022 22:04:32 +0800 Subject: [PATCH 039/137] Implement calnder in javascript --- src/lib/calendar.js | 995 ++++++++++++++++++++++++++++++++++++++++++++ src/lib/calendar.py | 1 - 2 files changed, 995 insertions(+), 1 deletion(-) create mode 100644 src/lib/calendar.js delete mode 100644 src/lib/calendar.py diff --git a/src/lib/calendar.js b/src/lib/calendar.js new file mode 100644 index 0000000000..8ea4926c77 --- /dev/null +++ b/src/lib/calendar.js @@ -0,0 +1,995 @@ +function $builtinmodule(name) { + const requiredImports = {}; + + const { + misceval: { chain: chainOrSuspend }, + importModule, + } = Sk; + + const importOrSuspend = (moduleName) => importModule(moduleName, false, true); + + return chainOrSuspend( + importOrSuspend("datetime"), + (datetime) => { + requiredImports.datetime = datetime; + return importOrSuspend("itertools"); + }, + (itertools) => { + requiredImports.iterRepeat = itertools.$d.repeat; + requiredImports.iterChain = itertools.$d.chain; + return calendarModule(requiredImports); + } + ); +} + +function calendarModule(required) { + const { + abstr: { setUpModuleMethods, numberBinOp, iter: pyIter, objectGetItem: pyGetItem }, + builtin: { + bool: pyBool, + bool: { true$: pyTrue, false$: pyFalse }, + func: pyFunc, + int_: pyInt, + list: pyList, + none: { none$: pyNone }, + str: pyStr, + slice: pySlice, + tuple: pyTuple, + range: pyRange, + max: pyMax, + min: pyMin, + property: pyProperty, + print: pyPrint, + enumerate: pyEnumerate, + ValueError, + }, + ffi: { remapToPy: toPy }, + misceval: { + isTrue, + iterator: pyIterator, + arrayFromIterable, + buildClass, + richCompareBool, + asIndexOrThrow, + objectRepr, + callsimArray: pyCall, + }, + global: skGlobal, + global: { strftime: strftimeJs }, + } = Sk; + + const _0 = new pyInt(0); + const _1 = new pyInt(1); + const _2 = new pyInt(2); + const _3 = new pyInt(3); + const _6 = new pyInt(6); + const _7 = new pyInt(7); + const _9 = new pyInt(9); + const _12 = new pyInt(12); + const _13 = new pyInt(13); + const _24 = new pyInt(24); + const _60 = new pyInt(60); + + const le = (a, b) => richCompareBool(a, b, "LtE"); + const ge = (a, b) => richCompareBool(a, b, "GtE"); + const eq = (a, b) => richCompareBool(a, b, "Eq"); + const mod = (a, b) => numberBinOp(a, b, "Mod"); + const add = (a, b) => numberBinOp(a, b, "Add"); + const sub = (a, b) => numberBinOp(a, b, "Sub"); + const mul = (a, b) => numberBinOp(a, b, "Mult"); + const inc = (a) => add(a, _1); + const dec = (a) => sub(a, _1); + const mod7 = (a) => mod(a, _7); + + const getA = (self, attr) => self.tp$getattr(new pyStr(attr)); + const callA = (self, attr, ...args) => pyCall(self.tp$getattr(new pyStr(attr)), args); + + function* iterJs(iterator) { + const it = pyIter(iterator); + let nxt; + while ((nxt = it.tp$iternext())) { + yield nxt; + } + } + + function iterFn(iter, fn) { + iter = pyIter(iter); + return new pyIterator(() => { + const nxt = iter.tp$iternext(); + return nxt && fn(nxt); + }, true); + } + + function makePyMethod(clsName, fn, { args, name, doc, defaults }) { + fn.co_varnames = ["self", ...(args || [])]; + fn.co_docstring = doc ? new pyStr(doc) : pyNone; + if (defaults) { + fn.$defaults = defaults; + } + fn.co_name = new pyStr(name); + fn.co_qualname = new pyStr(clsName + "." + name); + const rv = new pyFunc(fn); + rv.$module = calMod.__name__; + return rv; + } + + const { datetime, iterRepeat, iterChain } = required; + let { MINYEAR, MAXYEAR, date: pyDate } = datetime.$d; + + const centerMeth = getA(pyStr, "center"); + const pyCenter = (s, i) => pyCall(centerMeth, [s, i]); + const pyRStrip = (s) => new pyStr(s.toString().trimRight()); + + MINYEAR = MINYEAR.valueOf(); + MAXYEAR = MAXYEAR.valueOf(); + + const calMod = { + __name__: new pyStr("calendar"), + __all__: toPy([ + "IllegalMonthError", + "IllegalWeekdayError", + "setfirstweekday", + "firstweekday", + "isleap", + "leapdays", + "weekday", + "monthrange", + "monthcalendar", + "prmonth", + "month", + "prcal", + "calendar", + "timegm", + "month_name", + "month_abbr", + "day_name", + "day_abbr", + "Calendar", + "TextCalendar", + "HTMLCalendar", + "LocaleTextCalendar", + "LocaleHTMLCalendar", + "weekheader", + ]), + }; + + function makeErr(name, msg) { + return buildClass( + calMod, + (_gbl, loc) => { + loc.__init__ = new pyFunc(function __init__(self, attr) { + self.$attr = attr; + }); + loc.__str__ = new pyFunc(function __str__(self) { + return new pyStr(msg.replace("$", objectRepr(self.$attr))); + }); + }, + name, + [ValueError] + ); + } + + const IllegalMonthError = makeErr("IllegalMonthError", "bad month $; must be 1-12"); + const IllegalWeekdayError = makeErr( + "IllegalWeekdayError", + "bad weekday number $; must be 0 (Monday) to 6 (Sunday)" + ); + const January = 1; + const February = 2; + + const mdays = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; + + function mkLocalizedCls(fmtList, loc) { + loc.__init__ = new pyFunc(function __init__(self, format) { + self.format = format; + }); + + loc.__getitem__ = new pyFunc(function __getitem__(self, i) { + const funcs = pyGetItem(fmtList, i); + if (i instanceof pySlice) { + const rv = []; + for (const f of funcs.valueOf()) { + rv.push(pyCall(f, [self.format])); + } + return new pyList(rv); + } + return pyCall(funcs, [self.format]); + }); + + const len = new pyInt(fmtList.valueOf().length); + loc.__len__ = new pyFunc(function __len__(self) { + return len; + }); + } + + const _STRFTIME = new pyStr("strftime"); + + const _localized_month = buildClass( + calMod, + (_gbl, loc) => { + let _months = [new pyFunc((_x) => pyStr.$empty)]; + for (let i = 0; i < 12; i++) { + const d = new pyDate(2001, i + 1, 1); + _months.push(d.tp$getattr(_STRFTIME)); + } + _months = new pyList(_months); + loc._months = _months; + mkLocalizedCls(_months, loc); + }, + "_localized_month" + ); + + const _localized_day = buildClass( + calMod, + (_gbl, loc) => { + let _days = []; + for (let i = 0; i < 7; i++) { + const d = new pyDate(2001, 1, i + 1); + _days.push(d.tp$getattr(_STRFTIME)); + } + _days = new pyList(_days); + loc._days = _days; + mkLocalizedCls(_days, loc); + }, + "_localized_day" + ); + + // # Full and abbreviated names of weekdays + const day_name = pyCall(_localized_day, [new pyStr("%A")]); + const day_abbr = pyCall(_localized_day, [new pyStr("%a")]); + + // # Full and abbreviated names of months (1-based arrays!!!) + const month_name = pyCall(_localized_month, [new pyStr("%B")]); + const month_abbr = pyCall(_localized_month, [new pyStr("%b")]); + + const [MONDAY, TUESDAY, WEDNESDAY, THURSDAY, FRIDAY, SATURDAY, SUNDAY] = [0, 1, 2, 3, 4, 5, 6]; + + function isleap(year) { + year = asIndexOrThrow(year); + return year % 4 == 0 && (year % 100 != 0 || year % 400 === 0); + } + + function weekday(year, month, day) { + year = asIndexOrThrow(year); + if (!(MINYEAR <= year && year <= MAXYEAR)) { + year = 2000 + (year % 400); + } + const date = pyCall(pyDate, [new pyInt(year), month, day]); + return callA(pyDate, "weekday", date); + } + + function monthrange(year, month) { + if (!(le(_1, month) && le(month, _12))) { + throw pyCall(IllegalMonthError, [month]); + } + const day1 = weekday(year, month, _1); + month = asIndexOrThrow(month); + const ndays = mdays[month] + Number(month === February && isleap(year)); + return [day1, new pyInt(ndays)]; + } + + function _monthlen(year, month) { + month = asIndexOrThrow(month); + return new pyInt(mdays[month] + Number(month === February && isleap(year))); + } + + function _prevmonth(year, month) { + if (eq(month, _1)) { + return [dec(year), _12]; + } + return [year, dec(month)]; + } + + function _nextmonth(year, month) { + if (eq(month, _12)) { + return [inc(year), _1]; + } + return [year, inc(month)]; + } + + /***** Calendar Methods ******/ + + function iterweekdays(self) { + const iter = pyCall(pyRange, [self.fwd, add(self.fwd, _7)]); + return iterFn(iter, mod7); + } + + function itermonthdates(self, year, month) { + const iter = itermonthdays3(self, year, month); + return iterFn(iter, (ymdTuple) => pyCall(pyDate, ymdTuple.valueOf())); + } + + function itermonthdays(self, year, month) { + const [day1, ndays] = monthrange(year, month); + const days_before = mod7(sub(day1, self.fwd)); + const iter1 = pyCall(iterRepeat, [_0, days_before]); + const iter2 = pyCall(pyRange, [_1, inc(ndays)]); + const days_after = mod7(sub(self.fwd, add(day1, ndays))); + const iter3 = pyCall(iterRepeat, [_0, days_after]); + return pyCall(iterChain, [iter1, iter2, iter3]); + } + + function itermonthdays2(self, year, month) { + const iter = pyCall(pyEnumerate, [itermonthdays(self, year, month), self.fwd]); + return iterFn(iter, (nxt) => { + const [i, d] = nxt.valueOf(); + return new pyTuple([d, mod7(i)]); + }); + } + + function itermonthdays3(self, year, month) { + const ymdIter = (y, m, iter) => iterFn(iter, (nxt) => new pyTuple([y, m, nxt])); + const [day1, ndays] = monthrange(year, month); + const days_before = mod7(sub(day1, self.fwd)); + const days_after = mod7(sub(self.fwd, add(day1, ndays))); + const [y1, m1] = _prevmonth(year, month); + const end = inc(_monthlen(y1, m1)); + const iter1 = pyCall(pyRange, [sub(end, days_before), end]); + const iter2 = pyCall(pyRange, [_1, inc(ndays)]); + const [y3, m3] = _nextmonth(year, month); + const iter3 = pyCall(pyRange, [_1, inc(days_after)]); + + return pyCall(iterChain, [ymdIter(y1, m1, iter1), ymdIter(year, month, iter2), ymdIter(y3, m3, iter3)]); + } + + function itermonthdays4(self, year, month) { + const iter = itermonthdays3(self, year, month); + let i = 0; + return iterFn(iter, (nxt) => new pyTuple([...nxt.valueOf(), mod7(add(self.fwd, new pyInt(i++)))])); + } + + function _monthIter(meth, self, year, month) { + const arr = arrayFromIterable(meth(self, year, month)); + const rv = []; + for (let i = 0; i < arr.length; i += 7) { + rv.push(new pyList(arr.slice(i, i + 7))); + } + return new pyList(rv); + } + + function monthdatescalendar(self, year, month) { + return _monthIter(itermonthdates, self, year, month); + } + + function monthdays2calendar(self, year, month) { + return _monthIter(itermonthdays2, self, year, month); + } + + function monthdayscalendar(self, year, month) { + return _monthIter(itermonthdays, self, year, month); + } + + function _yearIter(meth, self, year, width) { + width = asIndexOrThrow(width); + const months = []; + for (let i = January; i < January + 12; i++) { + months.push(meth(self, year, new pyInt(i))); + } + const rv = []; + for (let i = 0; i < months.length; i += width) { + rv.push(new pyList(months.slice(i, i + width))); + } + return new pyList(rv); + } + + function yeardatescalendar(self, year, width) { + return _yearIter(monthdatescalendar, self, year, width); + } + + function yeardays2calendar(self, year, width) { + return _yearIter(monthdays2calendar, self, year, width); + } + + function yeardayscalendar(self, year, width) { + return _yearIter(monthdayscalendar, self, year, width); + } + + const Calendar = buildClass( + calMod, + (_gbl, loc) => { + function __init__(self, firstweekday) { + Object.defineProperty(self, "fwd", { + get() { + return mod7(this._fwd); + }, + set(val) { + this._fwd = val; + return true; + }, + }); + self.fwd = firstweekday; + return pyNone; + } + + function getfirstweekday(self) { + return self.fwd; + } + + function setfirstweekday(self, firstweekday) { + self.fwd = firstweekday; + return pyNone; + } + + const CalMeth = makePyMethod.bind(null, "Calendar"); + const FWD = ["firstweekday"]; + const YM = ["year", "month"]; + const YW = ["year", "width"]; + + const locals = { + __init__: CalMeth(__init__, { name: "__init__", args: FWD, defaults: [_0] }), + getfirstweekday: CalMeth(getfirstweekday, { name: "getfirstweekday" }), + setfirstweekday: CalMeth(setfirstweekday, { name: "setfirstweekday", args: FWD }), + iterweekdays: CalMeth(iterweekdays, { name: "iterweekdays" }), + itermonthdates: CalMeth(itermonthdates, { name: "itermonthdates", args: YM }), + itermonthdays: CalMeth(itermonthdays, { name: "itermonthdays", args: YM }), + itermonthdays2: CalMeth(itermonthdays2, { name: "itermonthdays2", args: YM }), + itermonthdays3: CalMeth(itermonthdays3, { name: "itermonthdays3", args: YM }), + itermonthdays4: CalMeth(itermonthdays4, { name: "itermonthdays4", args: YM }), + monthdatescalendar: CalMeth(monthdatescalendar, { name: "monthdatescalendar", args: YM }), + monthdays2calendar: CalMeth(monthdays2calendar, { name: "monthdays2calendar", args: YM }), + monthdayscalendar: CalMeth(monthdayscalendar, { name: "monthdayscalendar", args: YM }), + yeardatescalendar: CalMeth(yeardatescalendar, { name: "yeardatescalendar", args: YW, defaults: [_3] }), + yeardays2calendar: CalMeth(yeardays2calendar, { name: "yeardays2calendar", args: YW, defaults: [_3] }), + yeardayscalendar: CalMeth(yeardayscalendar, { name: "yeardayscalendar", args: YW, defaults: [_3] }), + }; + + locals.firstweekday = new pyProperty(locals.getfirstweekday, locals.setfirstweekday); + + Object.assign(loc, locals); + }, + "Calendar" + ); + + /***** TextCalendar *****/ + + function doTextFormatweekday(self, day, width) { + let names; + if (ge(width, _9)) { + names = day_name; + } else { + names = day_abbr; + } + return pyCenter(pyGetItem(pyGetItem(names, day), new pySlice(pyNone, width)), width); + } + + function doTextFormatmonthname(self, theyear, themonth, width, withyear = true) { + let s = pyGetItem(month_name, themonth); + if (isTrue(withyear)) { + s = mod(new pyStr("%s %r"), new pyTuple([s, theyear])); + } + return pyCenter(s, width); + } + + const TextCalendar = buildClass( + calMod, + (_gbl, loc) => { + const txtPrint = (x) => pyPrint([x], ["end", pyStr.$empty]); + + function prweek(self, theweek, width) { + txtPrint(callA(self, "formatweek", theweek, width)); + } + + function formatday(self, day, weekday, width) { + let s; + if (eq(day, _0)) { + s = pyStr.$empty; + } else { + s = mod(new pyStr("%2i"), day); + } + return pyCenter(s, width); + } + + function formatweek(self, theweek, width) { + const rv = []; + for (const dayWeekday of iterJs(theweek)) { + const [d, wd] = dayWeekday.valueOf(); + rv.push(callA(self, "formatday", d, wd, width).toString()); + } + return new pyStr(rv.join(" ")); + } + + const formatweekday = doTextFormatweekday; + + function formatweekheader(self, width) { + const rv = []; + for (const i of iterJs(iterweekdays(self))) { + rv.push(callA(self, "formatweekday", i, width).toString()); + } + return new pyStr(rv.join(" ")); + } + + const formatmonthname = doTextFormatmonthname; + + function prmonth(self, theyear, themonth, w, l) { + txtPrint(callA(self, "formatmonth", theyear, themonth, w, l)); + } + + function formatmonth(self, theyear, themonth, w, l) { + const addNewLines = (s) => new pyStr(s + "\n".repeat(l.valueOf())); + + w = pyMax([_2, w]); + l = pyMax([_1, l]); + let s = callA(self, "formatmonthname", theyear, themonth, dec(mul(_7, inc(w))), true); + s = pyRStrip(s); + s = addNewLines(s); + s = add(s, pyRStrip(callA(self, "formatweekheader", w))); + s = addNewLines(s); + for (const week of iterJs(monthdays2calendar(self, theyear, themonth))) { + s = add(s, pyRStrip(callA(self, "formatweek", week, w))); + s = addNewLines(s); + } + return s; + } + + function formatyear(self, theyear, w, l, c, m) { + w = pyMax([_2, w]); + l = pyMax([_1, l]); + c = pyMax([_2, c]); + const colwidth = dec(mul(inc(w), _7)); + let rv = ""; + const a = (s) => (rv += s); + a(pyRStrip(pyCenter(theyear.$r(), add(mul(colwidth, m), mul(c, dec(m)))))); + a("\n".repeat(l)); + const header = formatweekheader(self, w); + let i = 0; + for (const row of iterJs(yeardays2calendar(self, theyear, m))) { + const pyI = new pyInt(i); + const start = inc(mul(m, pyI)); + const end = pyMin([inc(mul(m, inc(pyI))), _13]); + const months = pyCall(pyRange, [start, end]); + a("\n".repeat(l)); + const names = iterFn(months, (k) => callA(self, "formatmonthname", theyear, k, colwidth, false)); + a(pyRStrip(formatstring(names, colwidth, c))); + a("\n".repeat(l)); + const headers = iterFn(months, (_k) => header); + a(pyRStrip(formatstring(headers, colwidth, c))); + a("\n".repeat(l)); + const height = Math.max(...row.valueOf().map((cal) => cal.valueOf().length)); + for (let j = 0; j < height; j++) { + const weeks = []; + for (let cal of row.valueOf()) { + cal = cal.valueOf(); + if (j >= cal.length) { + weeks.push(pyStr.$empty); + } else { + weeks.push(callA(self, "formatweek", cal[j], w)); + } + } + a(pyRStrip(formatstring(new pyList(weeks), colwidth, c))); + a("\n".repeat(l)); + } + i++; + } + return new pyStr(rv); + } + + function pryear(self, theyear, w, l, c, m) { + txtPrint(callA(self, "formatyear", theyear, w, l, c, m)); + } + + const TxtCalMethod = makePyMethod.bind(null, "TextCalendar"); + + const locals = { + prweek: TxtCalMethod(prweek, { name: "prweek", args: ["theweek", "width"] }), + formatday: TxtCalMethod(formatday, { name: "formatday", args: ["day", "weekday", "width"] }), + formatweek: TxtCalMethod(formatweek, { name: "formatweek", args: ["theweek", "width"] }), + formatweekday: TxtCalMethod(formatweekday, { name: "formatweekday", args: ["day", "width"] }), + formatweekheader: TxtCalMethod(formatweekheader, { name: "formatweekheader", args: ["width"] }), + formatmonthname: TxtCalMethod(formatmonthname, { + name: "formatmonthname", + args: ["theyear", "themonth", "width", "withyear"], + defaults: [pyTrue], + }), + prmonth: TxtCalMethod(prmonth, { + name: "prmonth", + args: ["theyear", "themonth", "w", "l"], + defaults: [_0, _0], + }), + formatmonth: TxtCalMethod(formatmonth, { + name: "formatmonth", + args: ["thyear", "themonth", "w", "l"], + defaults: [_0, _0], + }), + formatyear: TxtCalMethod(formatyear, { + name: "formatyear", + args: ["theyear", "w", "l", "c", "m"], + defaults: [_2, _1, _6, _3], + }), + pryear: TxtCalMethod(pryear, { + name: "pryear", + args: ["theyear", "w", "l", "c", "m"], + defaults: [_0, _0, _6, _3], + }), + }; + + Object.assign(loc, locals); + }, + "TextCalendar", + [Calendar] + ); + + /***** HTMLCalendar *****/ + + function doHtmlFormatweekday(self, day) { + return new pyStr( + `` + ); + } + + function doHtmlFormatmonthname(self, theyear, themonth, withyear = true) { + let s = "" + pyGetItem(month_name, themonth); + if (isTrue(withyear)) { + s += " " + theyear; + } + return new pyStr(``); + } + + const HTMLCalendar = buildClass( + calMod, + (_gbl, loc) => { + // # CSS classes for the day '); + const cellInMonth = new pyStr(''); + + function formatday(self, day, weekday) { + if (eq(day, _0)) { + return mod(cellOutsideMonth, getA(self, "cssclass_noday")); + } else { + return mod(cellInMonth, new pyTuple([pyGetItem(getA(self, "cssclasses"), weekday), day])); + } + } + + function formatweek(self, theweek) { + let rv = ""; + for (const nxt of iterJs(theweek)) { + const [d, wd] = nxt.valueOf(); + rv += callA(self, "formatday", d, wd); + } + return new pyStr(`${rv}`); + } + + const formatweekday = doHtmlFormatweekday; + + function formatweekheader(self) { + let rv = ""; + for (const i of iterJs(iterweekdays(self))) { + rv += callA(self, "formatweekday", i); + } + return new pyStr(`${rv}`); + } + + const formatmonthname = doHtmlFormatmonthname; + + function formatmonth(self, theyear, themonth, withyear = true) { + let rv = ""; + const a = (s) => (rv += s + "\n"); + a(`
%s
January
${pyGetItem(day_abbr, day)}
${s}
s + const cssclasses = toPy(["mon", "tue", "wed", "thu", "fri", "sat", "sun"]); + + // # CSS classes for the day s + const cssclasses_weekday_head = cssclasses; + + // # CSS class for the days before and after current month + const cssclass_noday = new pyStr("noday"); + + // # CSS class for the month's head + const cssclass_month_head = new pyStr("month"); + + // # CSS class for the month + const cssclass_month = cssclass_month_head; + + // # CSS class for the year's table head + const cssclass_year_head = new pyStr("year"); + + // # CSS class for the whole year table + const cssclass_year = cssclass_year_head; + + const cellOutsideMonth = new pyStr(' %d
`); + a(callA(self, "formatmonthname", theyear, themonth, withyear)); + a(formatweekheader(self)); + for (const week of iterJs(monthdays2calendar(self, theyear, themonth))) { + a(callA(self, "formatweek", week)); + } + a("
"); + return new pyStr(rv); + } + + function formatyear(self, theyear, width) { + let rv = ""; + const a = (s) => (rv += s); + width = pyMax([width, _1]).valueOf(); + a(``); + a("\n"); + a(``); + + for (let i = January; i < January + 12; i += width) { + a(""); + const end = Math.min(i + width, 13); + for (let m = i; m < end; m++) { + a(""); + } + a(""); + } + a("
${theyear}
"); + a(callA(self, "formatmonth", theyear, new pyInt(m), false)); + a("
"); + return new pyStr(rv); + } + + function formatyearpage(self, theyear, width = 3, css = "calendar.css", encoding = null) { + if (encoding === null || encoding === pyNone) { + encoding = new pyStr("utf-8"); + } + let rv = ""; + const a = (s) => (rv += s); + a(`\n`); + a( + '\n' + ); + a("\n"); + a("\n"); + a(`\n`); + if (css !== pyNone) { + a(`\n`); + } + a(`Calendar for ${theyear}\n`); + a("\n"); + a("\n"); + a(callA(self, "formatyear", theyear, width)); + a("\n"); + a("\n"); + return callA(pyStr, "encode", new pyStr(rv), encoding, new pyStr("ignore")); + } + + const HtmlMeth = makePyMethod.bind(null, "HTMLCalendar"); + + const locals = { + formatday: HtmlMeth(formatday, { name: "formatday", args: ["day", "weekday"] }), + formatweek: HtmlMeth(formatweek, { name: "formatweek", args: ["theweek"] }), + formatweekday: HtmlMeth(formatweekday, { name: "formatweekday", args: ["day"] }), + formatweekheader: HtmlMeth(formatweekheader, { name: "formatweekheader" }), + formatmonthname: HtmlMeth(formatmonthname, { + name: "formatmonthname", + args: ["theyear", "themonth", "withyear"], + defaults: [pyTrue], + }), + formatmonth: HtmlMeth(formatmonth, { + name: "formatmonth", + args: ["thyear", "themonth", "withyear"], + defaults: [pyTrue], + }), + formatyear: HtmlMeth(formatyear, { name: "formatyear", args: ["theyear", "width"], defaults: [_3] }), + formatyearpage: HtmlMeth(formatyearpage, { + name: "formatyearpage", + args: ["theyear", "width", "css", "encoding"], + defaults: [_3, new pyStr("calendar.css"), new pyStr("utf-8")], + }), + cssclasses, + cssclasses_weekday_head, + cssclass_noday, + cssclass_month_head, + cssclass_month, + cssclass_year_head, + cssclass_year, + }; + + Object.assign(loc, locals); + }, + "HTMLCalendar", + [Calendar] + ); + + /***** LocaleCalendars *****/ + + function withLocale(locale, fn) { + const s = strftimeJs.localizeByIdentifier(locale.toString()); + skGlobal.strftime = s; + try { + return fn(); + } finally { + skGlobal.strftime = strftimeJs; + } + } + + function localInit(self, locale) { + if (!isTrue(locale)) { + locale = new pyStr("en_US"); + } + self.locale = locale; + } + + const LocaleTextCalendar = buildClass( + calMod, + (_gbl, loc) => { + function __init__(self, firstweekday, locale) { + callA(TextCalendar, "__init__", self, firstweekday); + localInit(self, locale); + return pyNone; + } + function formatweekday(self, day, width) { + return withLocale(self.locale, () => doTextFormatweekday(self, day, width)); + } + function formatmonthname(self, theyear, themonth, width, withyear) { + return withLocale(self.locale, () => doTextFormatmonthname(self, theyear, themonth, width, withyear)); + } + const LocaleMeth = makePyMethod.bind(null, "LocaleTextCalendar"); + + const locals = { + __init__: LocaleMeth(__init__, { + name: "__init__", + args: ["firstweekday", "locale"], + defaults: [_0, pyNone], + }), + formatweekday: LocaleMeth(formatweekday, { name: "formatweekday", args: ["day", "width"] }), + formatmonthname: LocaleMeth(formatmonthname, { + name: "formatmonthname", + args: ["theyear", "themonth", "width", "withyear"], + defaults: [pyTrue], + }), + }; + + Object.assign(loc, locals); + }, + "LocaleTextCalendar", + [TextCalendar] + ); + + const LocaleHTMLCalendar = buildClass( + calMod, + (_gbl, loc) => { + function __init__(self, firstweekday, locale) { + callA(HTMLCalendar, "__init__", self, firstweekday); + localInit(self, locale); + return pyNone; + } + function formatweekday(self, day) { + return withLocale(self.locale, () => doHtmlFormatweekday(self, day)); + } + function formatmonthname(self, theyear, themonth, withyear) { + return withLocale(self.locale, () => doHtmlFormatmonthname(self, theyear, themonth, withyear)); + } + const LocaleMeth = makePyMethod.bind(null, "LocaleHTMLCalendar"); + + const locals = { + __init__: LocaleMeth(__init__, { + name: "__init__", + args: ["firstweekday", "locale"], + defaults: [_0, pyNone], + }), + formatweekday: LocaleMeth(formatweekday, { name: "formatweekday", args: ["day"] }), + formatmonthname: LocaleMeth(formatmonthname, { + name: "formatmonthname", + args: ["theyear", "themonth", "withyear"], + defaults: [pyTrue], + }), + }; + + Object.assign(loc, locals); + }, + "LocaleHTMLCalendar", + [HTMLCalendar] + ); + + const c = pyCall(TextCalendar, []); + + Object.assign(calMod, { + IllegalMonthError, + IllegalWeekdayError, + day_name, + month_name, + day_abbr, + month_abbr, + January: new pyInt(January), + February: new pyInt(February), + mdays: toPy(mdays), + MONDAY: new pyInt(MONDAY), + TUESDAY: new pyInt(TUESDAY), + WEDNESDAY: new pyInt(WEDNESDAY), + THURSDAY: new pyInt(THURSDAY), + FRIDAY: new pyInt(FRIDAY), + SATURDAY: new pyInt(SATURDAY), + SUNDAY: new pyInt(SUNDAY), + Calendar, + TextCalendar, + HTMLCalendar, + LocaleTextCalendar, + LocaleHTMLCalendar, + c, + firstweekday: getA(c, "getfirstweekday"), + monthcalendar: getA(c, "monthdayscalendar"), + prweek: getA(c, "prweek"), + week: getA(c, "formatweek"), + weekheader: getA(c, "formatweekheader"), + prmonth: getA(c, "prmonth"), + month: getA(c, "formatmonth"), + calendar: getA(c, "formatyear"), + prcal: getA(c, "pryear"), + }); + + // # Spacing of month columns for multi-column year calendar + const _colwidth = new pyInt(7 * 3 - 1); //# Amount printed by prweek() + const _spacing = _6; //# Number of spaces between columns + + function format(cols, colwidth, spacing) { + pyPrint([formatstring(cols, colwidth, spacing)]); + return pyNone; + } + + function formatstring(cols, colwidth, spacing) { + colwidth || (colwidth = _colwidth); + spacing || (spacing = _spacing); + spacing = mul(spacing, new pyStr(" ")); + const rv = []; + for (const c of iterJs(cols)) { + rv.push(pyCenter(c, colwidth).toString()); + } + return new pyStr(rv.join(spacing.toString())); + } + + const EPOCH = 1970; + const toOrd = getA(pyDate, "toordinal"); + const _EPOCH_ORD = pyCall(toOrd, [new pyDate(EPOCH, 1, 1)]); + + setUpModuleMethods("calendar", calMod, { + isleap: { + $meth(year) { + return pyBool(isleap(year)); + }, + $flags: { NamedArgs: ["year"] }, + $doc: "Return True for leap years, False for non-leap years", + }, + leapdays: { + $meth(y1, y2) { + y1 = asIndexOrThrow(y1) - 1; + y2 = asIndexOrThrow(y2) - 1; + const _ = Math.floor; + return new pyInt(_(y2 / 4) - _(y1 / 4) - (_(y2 / 100) - _(y1 / 100)) + (_(y2 / 400) - _(y1 / 400))); + }, + $flags: { MinArgs: 2, MaxArgs: 2 }, + }, + weekday: { + $meth: weekday, + $flags: { NamedArgs: ["year", "month", "day"] }, + $doc: "Return weekday (0-6 ~ Mon-Sun) for year, month (1-12), day (1-31).", + }, + monthrange: { + $meth(year, month) { + return new pyTuple(monthrange(year, month)); + }, + $flags: { NamedArgs: ["year", "month"] }, + $doc: "Return weekday (0-6 ~ Mon-Sun) and number of days (28-31) for year, month.", + }, + setfirstweekday: { + $meth(firstweekday) { + const jsweekday = asIndexOrThrow(firstweekday); + if (!(MONDAY <= jsweekday && jsweekday <= SUNDAY)) { + throw pyCall(IllegalWeekdayError, [firstweekday]); + } + c.fwd = firstweekday; + }, + $flags: { NamedArgs: ["firstweekday"] }, + }, + format: { + $meth: format, + $flags: { NamedArgs: ["cols", "colwidth", "spacing"], Defaults: [_colwidth, _spacing] }, + }, + formatstring: { + $meth: formatstring, + $flags: { NamedArgs: ["cols", "colwidth", "spacing"], Defaults: [_colwidth, _spacing] }, + }, + timegm: { + $meth(tuple) { + const [year, month, day, hour, minute, second] = tuple.valueOf(); + const date = pyCall(pyDate, [year, month, _1]); + const asOrd = pyCall(toOrd, [date]); + const days = add(sub(asOrd, _EPOCH_ORD), dec(day)); + const hours = add(mul(days, _24), hour); + const minutes = add(mul(hours, _60), minute); + const seconds = add(mul(minutes, _60), second); + return seconds; + }, + $flags: { OneArg: true }, + }, + }); + + return calMod; +} diff --git a/src/lib/calendar.py b/src/lib/calendar.py deleted file mode 100644 index bb39aecab1..0000000000 --- a/src/lib/calendar.py +++ /dev/null @@ -1 +0,0 @@ -import _sk_fail; _sk_fail._("calendar") From a6beb53ae4540c82150c96dfa3d2802329df4922 Mon Sep 17 00:00:00 2001 From: stu Date: Sun, 3 Apr 2022 22:04:55 +0800 Subject: [PATCH 040/137] add valueOf methods to simple types --- src/bool.js | 5 ++++- src/float.js | 5 +++++ src/int.js | 3 +++ src/list.js | 3 +++ src/nonetype.js | 5 +++++ src/str.js | 13 ++++++------- src/tuple.js | 3 +++ 7 files changed, 29 insertions(+), 8 deletions(-) diff --git a/src/bool.js b/src/bool.js index 6b65e0d1e8..d6d2ef836b 100644 --- a/src/bool.js +++ b/src/bool.js @@ -70,7 +70,10 @@ Sk.builtin.bool = Sk.abstr.buildNativeClass("bool", { proto: { str$False: new Sk.builtin.str("False"), str$True: new Sk.builtin.str("True"), - } + valueOf() { + return !!this.v; + }, + }, }); Sk.exportSymbol("Sk.builtin.bool", Sk.builtin.bool); diff --git a/src/float.js b/src/float.js index 1235614129..a907f3173b 100644 --- a/src/float.js +++ b/src/float.js @@ -242,6 +242,11 @@ Sk.builtin.float_ = Sk.abstr.buildNativeClass("float", { $doc: Sk.builtin.none.none$, }, }, + proto: { + valueOf() { + return this.v; + } + } }); function frexp(arg) { diff --git a/src/int.js b/src/int.js index aedf684b70..a8db4438b6 100644 --- a/src/int.js +++ b/src/int.js @@ -322,6 +322,9 @@ Sk.builtin.int_ = Sk.abstr.buildNativeClass("int", { const result = bankRound * multiplier; return new Sk.builtin.int_(result); }, + valueOf() { + return this.v; + }, }, }); diff --git a/src/list.js b/src/list.js index 38aeb4fd07..36b59abedd 100644 --- a/src/list.js +++ b/src/list.js @@ -381,6 +381,9 @@ Sk.builtin.list = Sk.abstr.buildNativeClass("list", { dec += offdir; }); }, + valueOf() { + return this.v; + }, }, }); diff --git a/src/nonetype.js b/src/nonetype.js index 604078cc22..59564b3ca5 100644 --- a/src/nonetype.js +++ b/src/nonetype.js @@ -23,6 +23,11 @@ Sk.builtin.none = Sk.abstr.buildNativeClass("NoneType", { return false; }, }, + proto: { + valueOf() { + return null; + } + }, flags: { sk$unacceptableBase: true, }, diff --git a/src/str.js b/src/str.js index 76c7b87ae7..67158437d6 100755 --- a/src/str.js +++ b/src/str.js @@ -287,6 +287,9 @@ Sk.builtin.str = Sk.abstr.buildNativeClass("str", { } throw new Sk.builtin.TypeError("a str instance is required not '" + Sk.abstr.typeName(tgt) + "'"); }, + valueOf() { + return this.v; + }, $isIdentifier() { return Sk.token.isIdentifier(this.v); }, @@ -940,13 +943,9 @@ function mkJust(isRight, isCenter) { if (mylen >= len) { return new Sk.builtin.str(this.v); } else if (isCenter) { - newstr = fillchar.repeat(Math.floor((len - mylen) / 2)); - newstr = newstr + this.v + newstr; - - if ((len - mylen) % 2) { - newstr += fillchar; - } - + const marg = len - mylen; + const left = Math.floor(marg / 2) + (marg & len & 1); + newstr = fillchar.repeat(left) + this.v + fillchar.repeat(marg - left); return new Sk.builtin.str(newstr); } else { newstr = fillchar.repeat(len - mylen); diff --git a/src/tuple.js b/src/tuple.js index fd14dcb298..6d1bcd3ef4 100644 --- a/src/tuple.js +++ b/src/tuple.js @@ -141,6 +141,9 @@ Sk.builtin.tuple = Sk.abstr.buildNativeClass("tuple", { sk$asarray() { return this.v.slice(0); }, + valueOf() { + return this.v; + }, }, methods: /**@lends {Sk.builtin.tuple.prototype}*/ { __getnewargs__: { From 03ac134538b2a0e2bda0e099e7738bb8ff666ee2 Mon Sep 17 00:00:00 2001 From: stu Date: Sun, 3 Apr 2022 22:05:11 +0800 Subject: [PATCH 041/137] fix a couple of minor issues --- src/builtin/sys.js | 2 ++ src/lib/unittest/__init__.py | 3 ++- src/misceval.js | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/builtin/sys.js b/src/builtin/sys.js index 5afe9367ca..92cc535418 100644 --- a/src/builtin/sys.js +++ b/src/builtin/sys.js @@ -32,6 +32,8 @@ var $builtinmodule = function (name) { sys.path = Sk.realsyspath; + sys.getdefaultencoding = new Sk.builtin.func(() => new Sk.builtin.str("utf-8")); + sys.getExecutionLimit = new Sk.builtin.func(function () { if (Sk.execLimit === null) { return Sk.builtin.none.none$; diff --git a/src/lib/unittest/__init__.py b/src/lib/unittest/__init__.py index 3e1cce721c..e3f220373d 100644 --- a/src/lib/unittest/__init__.py +++ b/src/lib/unittest/__init__.py @@ -288,7 +288,8 @@ def fail(self, msg=None): self.assertFailed += 1 def showSummary(self): - pct = self.numPassed / (self.numPassed+self.numFailed) * 100 + # don't divde by zero + # pct = self.numPassed / (self.numPassed+self.numFailed) * 100 print("Ran %d tests, passed: %d failed: %d\n" % (self.numPassed+self.numFailed, self.numPassed, self.numFailed)) diff --git a/src/misceval.js b/src/misceval.js index da3b192df0..5721456ab5 100644 --- a/src/misceval.js +++ b/src/misceval.js @@ -248,7 +248,7 @@ Sk.misceval.iterator = Sk.abstr.buildIteratorClass("iterator", { constructor : function iterator (fn, handlesOwnSuspensions) { this.tp$iternext = handlesOwnSuspensions ? fn : function (canSuspend) { let x = fn(); - if (canSuspend || !x.$isSuspension) { + if (canSuspend || x === undefined || !x.$isSuspension) { return x; } else { return Sk.misceval.retryOptionalSuspensionOrThrow(x); From 65719dd51db24711b0fe8deb6f1166a4d7d7c7c8 Mon Sep 17 00:00:00 2001 From: stu Date: Thu, 7 Jul 2022 10:00:46 +0800 Subject: [PATCH 042/137] make sure buildClass accepts undefined bases --- src/misceval.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/misceval.js b/src/misceval.js index 5721456ab5..6032b637d5 100644 --- a/src/misceval.js +++ b/src/misceval.js @@ -1334,6 +1334,7 @@ Sk.misceval.buildClass = function (globals, func, name, bases, cell, kws) { const _bases = update_bases(bases); // todo this function should go through the bases and check for __mro_entries__ kws = kws || []; + bases = bases || []; let meta; let is_class = true; const meta_idx = kws.indexOf("metaclass"); From 577aeef780494b9c88c7cf06cbf7b70a3d9170e3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 Jul 2022 00:13:23 +0000 Subject: [PATCH 043/137] Bump shelljs from 0.8.4 to 0.8.5 Bumps [shelljs](https://github.com/shelljs/shelljs) from 0.8.4 to 0.8.5. - [Release notes](https://github.com/shelljs/shelljs/releases) - [Changelog](https://github.com/shelljs/shelljs/blob/master/CHANGELOG.md) - [Commits](https://github.com/shelljs/shelljs/compare/v0.8.4...v0.8.5) --- updated-dependencies: - dependency-name: shelljs dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index e3fad774bd..187acc5a61 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,7 +32,7 @@ "open": "^6.3.0", "readline-sync": "^1.4.9", "setimmediate": "^1.0.5", - "shelljs": "^0.8.3", + "shelljs": "^0.8.5", "strftime": "^0.10.1", "terser": "^5.7.0", "webpack": "^4.32.0", @@ -5328,9 +5328,9 @@ } }, "node_modules/shelljs": { - "version": "0.8.4", - "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.4.tgz", - "integrity": "sha512-7gk3UZ9kOfPLIAbslLzyWeGiEqx9e3rxwZM0KE6EL8GlGwjym9Mrlx5/p33bWTu9YG6vcS4MBxYZDHYr5lr8BQ==", + "version": "0.8.5", + "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz", + "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==", "dev": true, "dependencies": { "glob": "^7.0.0", @@ -11203,9 +11203,9 @@ "dev": true }, "shelljs": { - "version": "0.8.4", - "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.4.tgz", - "integrity": "sha512-7gk3UZ9kOfPLIAbslLzyWeGiEqx9e3rxwZM0KE6EL8GlGwjym9Mrlx5/p33bWTu9YG6vcS4MBxYZDHYr5lr8BQ==", + "version": "0.8.5", + "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz", + "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==", "dev": true, "requires": { "glob": "^7.0.0", diff --git a/package.json b/package.json index 7460158162..4e1fc98132 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "open": "^6.3.0", "readline-sync": "^1.4.9", "setimmediate": "^1.0.5", - "shelljs": "^0.8.3", + "shelljs": "^0.8.5", "strftime": "^0.10.1", "terser": "^5.7.0", "webpack": "^4.32.0", From 3f3466092282a11525cfe11d7afe151c506744e2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 Jul 2022 00:16:28 +0000 Subject: [PATCH 044/137] build(deps): bump ansi-regex from 3.0.0 to 3.0.1 Bumps [ansi-regex](https://github.com/chalk/ansi-regex) from 3.0.0 to 3.0.1. - [Release notes](https://github.com/chalk/ansi-regex/releases) - [Commits](https://github.com/chalk/ansi-regex/compare/v3.0.0...v3.0.1) --- updated-dependencies: - dependency-name: ansi-regex dependency-type: indirect ... Signed-off-by: dependabot[bot] --- package-lock.json | 72 +++++++++++++++++++++++------------------------ 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/package-lock.json b/package-lock.json index 187acc5a61..45c7635cc8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -390,9 +390,9 @@ } }, "node_modules/ansi-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", - "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.1.tgz", + "integrity": "sha512-+O9Jct8wf++lXxxFc4hc8LsjaSq0HFzzL7cVsw8pRDIPdjKD2mT4ytDZlLuSBZ4cLKZFXIrMGO7DbQCtMJJMKw==", "dev": true, "engines": { "node": ">=4" @@ -1171,9 +1171,9 @@ } }, "node_modules/cliui/node_modules/ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz", + "integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==", "dev": true, "engines": { "node": ">=6" @@ -3216,9 +3216,9 @@ } }, "node_modules/inquirer/node_modules/ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz", + "integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==", "dev": true, "engines": { "node": ">=6" @@ -5758,9 +5758,9 @@ } }, "node_modules/table/node_modules/ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz", + "integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==", "dev": true, "engines": { "node": ">=6" @@ -6592,9 +6592,9 @@ } }, "node_modules/wrap-ansi/node_modules/ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz", + "integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==", "dev": true, "engines": { "node": ">=6" @@ -6690,9 +6690,9 @@ } }, "node_modules/yargs/node_modules/ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz", + "integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==", "dev": true, "engines": { "node": ">=6" @@ -7068,9 +7068,9 @@ "dev": true }, "ansi-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", - "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.1.tgz", + "integrity": "sha512-+O9Jct8wf++lXxxFc4hc8LsjaSq0HFzzL7cVsw8pRDIPdjKD2mT4ytDZlLuSBZ4cLKZFXIrMGO7DbQCtMJJMKw==", "dev": true }, "ansi-styles": { @@ -7745,9 +7745,9 @@ }, "dependencies": { "ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz", + "integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==", "dev": true }, "string-width": { @@ -9471,9 +9471,9 @@ }, "dependencies": { "ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz", + "integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==", "dev": true }, "strip-ansi": { @@ -11565,9 +11565,9 @@ }, "dependencies": { "ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz", + "integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==", "dev": true }, "string-width": { @@ -12273,9 +12273,9 @@ }, "dependencies": { "ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz", + "integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==", "dev": true }, "string-width": { @@ -12358,9 +12358,9 @@ }, "dependencies": { "ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz", + "integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==", "dev": true }, "camelcase": { From 8b9b405ec7c442ba2aa854c8147215d0979d9478 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 Jul 2022 00:21:15 +0000 Subject: [PATCH 045/137] build(deps-dev): bump ejs from 2.7.4 to 3.1.7 Bumps [ejs](https://github.com/mde/ejs) from 2.7.4 to 3.1.7. - [Release notes](https://github.com/mde/ejs/releases) - [Changelog](https://github.com/mde/ejs/blob/main/CHANGELOG.md) - [Commits](https://github.com/mde/ejs/compare/v2.7.4...v3.1.7) --- updated-dependencies: - dependency-name: ejs dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- package-lock.json | 247 ++++++++++++++++++++++++++++++++++++++++++++-- package.json | 2 +- 2 files changed, 240 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index 45c7635cc8..feceba924d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,7 +19,7 @@ "commander": "^2.20.0", "compression-webpack-plugin": "^2.0.0", "copy-webpack-plugin": "^5.1.1", - "ejs": "^2.6.1", + "ejs": "^3.1.7", "eslint": "^5.16.0", "eslint-loader": "^2.1.2", "express": "^4.17.0", @@ -563,6 +563,12 @@ "node": ">=4" } }, + "node_modules/async": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", + "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==", + "dev": true + }, "node_modules/async-each": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/async-each/-/async-each-1.0.3.tgz", @@ -1844,10 +1850,16 @@ "dev": true }, "node_modules/ejs": { - "version": "2.7.4", - "resolved": "https://registry.npmjs.org/ejs/-/ejs-2.7.4.tgz", - "integrity": "sha512-7vmuyh5+kuUyJKePhQfRQBhXV5Ce+RnaeeQArKu1EAMpL3WbgMt5WG6uQZpEVvYSSsxMXRKOewtDk9RaTKXRlA==", + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.7.tgz", + "integrity": "sha512-BIar7R6abbUxDA3bfXrO4DSgwo8I+fB5/1zgujl3HLLjwd6+9iOnrT+t3grn2qbk9vOgBubXOFwX2m9axoFaGw==", "dev": true, + "dependencies": { + "jake": "^10.8.5" + }, + "bin": { + "ejs": "bin/cli.js" + }, "engines": { "node": ">=0.10.0" } @@ -2486,6 +2498,36 @@ "node": ">=4" } }, + "node_modules/filelist": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", + "integrity": "sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==", + "dev": true, + "dependencies": { + "minimatch": "^5.0.1" + } + }, + "node_modules/filelist/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/filelist/node_modules/minimatch": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.0.tgz", + "integrity": "sha512-9TPBGGak4nHfGZsPBohm9AWg6NoT7QTCehS3BIJABslyZbzxfV78QM2Y6+i741OPZIafFAaiiEMh5OyIrJPgtg==", + "dev": true, + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/fill-range": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz", @@ -3500,6 +3542,94 @@ "node": ">=0.10.0" } }, + "node_modules/jake": { + "version": "10.8.5", + "resolved": "https://registry.npmjs.org/jake/-/jake-10.8.5.tgz", + "integrity": "sha512-sVpxYeuAhWt0OTWITwT98oyV0GsXyMlXCF+3L1SuafBVUIr/uILGRB+NqwkzhgXKvoJpDIpQvqkUALgdmQsQxw==", + "dev": true, + "dependencies": { + "async": "^3.2.3", + "chalk": "^4.0.2", + "filelist": "^1.0.1", + "minimatch": "^3.0.4" + }, + "bin": { + "jake": "bin/cli.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/jake/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/jake/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/jake/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/jake/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/jake/node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/jake/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/js-beautify": { "version": "1.13.5", "resolved": "https://registry.npmjs.org/js-beautify/-/js-beautify-1.13.5.tgz", @@ -7212,6 +7342,12 @@ "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==", "dev": true }, + "async": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", + "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==", + "dev": true + }, "async-each": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/async-each/-/async-each-1.0.3.tgz", @@ -8332,10 +8468,13 @@ "dev": true }, "ejs": { - "version": "2.7.4", - "resolved": "https://registry.npmjs.org/ejs/-/ejs-2.7.4.tgz", - "integrity": "sha512-7vmuyh5+kuUyJKePhQfRQBhXV5Ce+RnaeeQArKu1EAMpL3WbgMt5WG6uQZpEVvYSSsxMXRKOewtDk9RaTKXRlA==", - "dev": true + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.7.tgz", + "integrity": "sha512-BIar7R6abbUxDA3bfXrO4DSgwo8I+fB5/1zgujl3HLLjwd6+9iOnrT+t3grn2qbk9vOgBubXOFwX2m9axoFaGw==", + "dev": true, + "requires": { + "jake": "^10.8.5" + } }, "elliptic": { "version": "6.5.4", @@ -8871,6 +9010,35 @@ "flat-cache": "^2.0.1" } }, + "filelist": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", + "integrity": "sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==", + "dev": true, + "requires": { + "minimatch": "^5.0.1" + }, + "dependencies": { + "brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "requires": { + "balanced-match": "^1.0.0" + } + }, + "minimatch": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.0.tgz", + "integrity": "sha512-9TPBGGak4nHfGZsPBohm9AWg6NoT7QTCehS3BIJABslyZbzxfV78QM2Y6+i741OPZIafFAaiiEMh5OyIrJPgtg==", + "dev": true, + "requires": { + "brace-expansion": "^2.0.1" + } + } + } + }, "fill-range": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz", @@ -9693,6 +9861,69 @@ "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=", "dev": true }, + "jake": { + "version": "10.8.5", + "resolved": "https://registry.npmjs.org/jake/-/jake-10.8.5.tgz", + "integrity": "sha512-sVpxYeuAhWt0OTWITwT98oyV0GsXyMlXCF+3L1SuafBVUIr/uILGRB+NqwkzhgXKvoJpDIpQvqkUALgdmQsQxw==", + "dev": true, + "requires": { + "async": "^3.2.3", + "chalk": "^4.0.2", + "filelist": "^1.0.1", + "minimatch": "^3.0.4" + }, + "dependencies": { + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + } + } + }, "js-beautify": { "version": "1.13.5", "resolved": "https://registry.npmjs.org/js-beautify/-/js-beautify-1.13.5.tgz", diff --git a/package.json b/package.json index 4e1fc98132..122073dac1 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ "commander": "^2.20.0", "compression-webpack-plugin": "^2.0.0", "copy-webpack-plugin": "^5.1.1", - "ejs": "^2.6.1", + "ejs": "^3.1.7", "eslint": "^5.16.0", "eslint-loader": "^2.1.2", "express": "^4.17.0", From 797ea4123432ed9f9083ca337230e41019f7f822 Mon Sep 17 00:00:00 2001 From: alexwenbj Date: Fri, 8 Jul 2022 10:34:11 +0800 Subject: [PATCH 046/137] add missing global attributes to turtle See #1393. close #1392 --- src/lib/turtle.js | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/lib/turtle.js b/src/lib/turtle.js index 8962aad2f7..f1ac0e4bbd 100644 --- a/src/lib/turtle.js +++ b/src/lib/turtle.js @@ -2339,6 +2339,21 @@ function generateTurtleModule(_target) { addModuleMethod(Screen, _module, "$window_width", getScreen); addModuleMethod(Screen, _module, "$window_height", getScreen); addModuleMethod(Screen, _module, "$title", getScreen); + + addModuleMethod(Screen, _module, "$onkey", getScreen); + addModuleMethod(Screen, _module, "$listen", getScreen); + addModuleMethod(Screen, _module, "$register_shape", getScreen); + addModuleMethod(Screen, _module, "$clearscreen", getScreen); + addModuleMethod(Screen, _module, "$bgcolor", getScreen); + addModuleMethod(Screen, _module, "$bgpic", getScreen); + addModuleMethod(Screen, _module, "$setworldcoordinates", getScreen); + addModuleMethod(Screen, _module, "$ontimer", getScreen); + addModuleMethod(Screen, _module, "$onscreenclick", getScreen); + addModuleMethod(Screen, _module, "$exitonclick", getScreen); + + addModuleMethod(Screen, _module, "$resetscreen", getScreen); + addModuleMethod(Screen, _module, "$setup", getScreen); + addModuleMethod(Screen, _module, "$turtles", getScreen); _module.Turtle = Sk.misceval.buildClass(_module, TurtleWrapper, "Turtle", []); _module.Screen = Sk.misceval.buildClass(_module, ScreenWrapper, "Screen", []); From ae9a36f4e32e29c497517bfe59aeed2c58709a37 Mon Sep 17 00:00:00 2001 From: stu Date: Sat, 30 Jul 2022 13:20:19 +0800 Subject: [PATCH 047/137] More descriptor suspension awareness --- src/descr.js | 10 ++++++---- src/property_class_static.js | 8 +++++--- src/slotdefs.js | 4 ++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/descr.js b/src/descr.js index 5323888499..20766d2be5 100644 --- a/src/descr.js +++ b/src/descr.js @@ -100,24 +100,26 @@ Sk.builtin.getset_descriptor = buildDescriptor("getset_descriptor", undefined, { this.d$name = getset_def.$name; }, slots: { - tp$descr_get(obj, type) { + tp$descr_get(obj, type, canSuspend) { let ret; if ((ret = this.d$check(obj))) { return ret; } if (this.$get !== undefined) { - return this.$get.call(obj); + const rv = this.$get.call(obj); + return canSuspend ? rv : Sk.misceval.retryOptionalSuspensionOrThrow(rv); } throw new Sk.builtin.AttributeError( "getset_descriptor '" + this.d$name + "' of '" + this.d$type.prototype.tp$name + "' objects is not readable" ); }, - tp$descr_set(obj, value) { + tp$descr_set(obj, value, canSuspend) { this.d$set_check(obj); if (this.$set !== undefined) { - return this.$set.call(obj, value); + const rv = this.$set.call(obj, value); + return canSuspend ? rv : Sk.misceval.retryOptionalSuspensionOrThrow(rv); } throw new Sk.builtin.AttributeError("attribute '" + this.d$name + "' of '" + this.d$type.prototype.tp$name + "' objects is readonly"); }, diff --git a/src/property_class_static.js b/src/property_class_static.js index 819b82813c..c192f53cf4 100644 --- a/src/property_class_static.js +++ b/src/property_class_static.js @@ -49,7 +49,7 @@ Sk.builtin.property = Sk.abstr.buildNativeClass("property", { const rv = Sk.misceval.callsimOrSuspendArray(this.prop$get, [obj]); return canSuspend ? rv : Sk.misceval.retryOptionalSuspensionOrThrow(rv); }, - tp$descr_set(obj, value) { + tp$descr_set(obj, value, canSuspend) { let func; if (value == null) { func = this.prop$del; @@ -64,11 +64,13 @@ Sk.builtin.property = Sk.abstr.buildNativeClass("property", { throw new Sk.builtin.TypeError("'" + Sk.abstr.typeName(func) + "' is not callable"); } + let rv; if (value == null) { - return func.tp$call([obj]); + rv = func.tp$call([obj]); } else { - return func.tp$call([obj, value]); + rv = func.tp$call([obj, value]); } + return canSuspend ? rv : Sk.misceval.retryOptionalSuspensionOrThrow(rv); }, }, methods: { diff --git a/src/slotdefs.js b/src/slotdefs.js index d149f11fa4..f85988e4ab 100644 --- a/src/slotdefs.js +++ b/src/slotdefs.js @@ -208,9 +208,9 @@ function slotFuncSetDelete(set_name, del_name, error_msg) { return func.d$wrapped.call(this, pyObject, value); } if (func.tp$descr_get) { - func = func.tp$descr_get(this); + func = func.tp$descr_get(this, this.ob$type, canSuspend); } - + if (func !== undefined) { const args = value === undefined ? [pyObject] : [pyObject, value]; res = Sk.misceval.callsimOrSuspendArray(func, args); From 7fe11a1b2ef9460d2110e6ed24067fb306f0de5a Mon Sep 17 00:00:00 2001 From: stu Date: Thu, 18 Aug 2022 13:02:50 +0800 Subject: [PATCH 048/137] fix frozenset has a non writable property --- src/set.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/set.js b/src/set.js index 9f618bd6de..40d8ffebe0 100644 --- a/src/set.js +++ b/src/set.js @@ -532,10 +532,7 @@ Sk.builtin.frozenset = Sk.abstr.buildNativeClass("frozenset", { ), }); -Sk.builtin.frozenset.$emptyset = Object.create(Sk.builtin.frozenset.prototype, { - v: { value: new Sk.builtin.dict([]), enumerable: true }, - in$repr: { value: false, enumerable: true }, -}); +Sk.builtin.frozenset.$emptyset = new Sk.builtin.frozenset([]); Sk.exportSymbol("Sk.builtin.frozenset", Sk.builtin.frozenset); From 879b72a624438d3008c09c7c4ad503ee6313394a Mon Sep 17 00:00:00 2001 From: stu Date: Sun, 4 Sep 2022 10:28:18 +0100 Subject: [PATCH 049/137] fix: time.localtime has wrong offset --- src/lib/time.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/time.js b/src/lib/time.js index e7dc96f9ea..f1a57fbd11 100644 --- a/src/lib/time.js +++ b/src/lib/time.js @@ -150,7 +150,7 @@ var $builtinmodule = function (name) { if (utc) { tm_info = [new Sk.builtin.str("UTC"),new Sk.builtin.int_(0) ] } else { - var offset = -(stdTimezoneOffset())/60; + var offset = -date.getTimezoneOffset() / 60; var pad = offset < 0 ? "-" : "+"; var tm_zone = pad + ("" + Math.abs(offset)).padStart(2, "0"); tm_info = [new Sk.builtin.str(tm_zone), new Sk.builtin.int_(offset * 3600)]; From f2c18d1df8448e428403c712651ada0d8768356f Mon Sep 17 00:00:00 2001 From: stu Date: Fri, 4 Nov 2022 13:58:04 +0800 Subject: [PATCH 050/137] pr: account for a python string being a key in a skulpt keyword array. They should be JavaScript strings, but a python string may have leaked here --- src/abstract.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/abstract.js b/src/abstract.js index 3a4c4f5f09..c8df64bbce 100644 --- a/src/abstract.js +++ b/src/abstract.js @@ -619,7 +619,7 @@ Sk.abstr.copyKeywordsToNamedArgs = function (func_name, varnames, args, kwargs, args = args.slice(0); // make a copy of args for (let i = 0; i < kwargs.length; i += 2) { - const name = kwargs[i]; // JS string + const name = kwargs[i].toString(); // JS string but account for python string const value = kwargs[i + 1]; // Python value const idx = varnames.indexOf(name); From afef2cf97f9af5f2d6e5c6c73687354f3d5d8652 Mon Sep 17 00:00:00 2001 From: stu Date: Thu, 3 Nov 2022 13:20:00 +0800 Subject: [PATCH 051/137] Add Failing test for #1470 --- test/unit3/test_skulpt_bugs.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/unit3/test_skulpt_bugs.py b/test/unit3/test_skulpt_bugs.py index 28d306365b..15136f6a4b 100644 --- a/test/unit3/test_skulpt_bugs.py +++ b/test/unit3/test_skulpt_bugs.py @@ -23,6 +23,23 @@ def test_bug_1345(self): except Exception: self.fail("this shouldn't fail") +class TestRegressions(unittest.TestCase): + def test_bug_1470(self): + global i + i = 0 + + def g(): + global i + i += 1 + + def f(x): + pass + + f(x=g()) + + self.assertEqual(i, 1) + + if __name__ == "__main__": unittest.main() From 22a8f746f8af561a5613ac0058e5ff0716b0a35e Mon Sep 17 00:00:00 2001 From: stu Date: Thu, 3 Nov 2022 13:20:16 +0800 Subject: [PATCH 052/137] fix repeated keyword unpacking in compile.js --- src/compile.js | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/src/compile.js b/src/compile.js index 6ca7b38df1..bb8d63bb7d 100644 --- a/src/compile.js +++ b/src/compile.js @@ -685,32 +685,6 @@ Compiler.prototype.ccall = function (e) { let positionalArgs = this.cunpackstarstoarray(e.args, !Sk.__future__.python3); let keywordArgs = this.cunpackkwstoarray(e.keywords, func); - if (e.keywords && e.keywords.length > 0) { - let hasStars = false; - kwarray = []; - for (let kw of e.keywords) { - if (hasStars && !Sk.__future__.python3) { - throw new Sk.builtin.SyntaxError("Advanced unpacking of function arguments is not supported in Python 2"); - } - if (kw.arg) { - kwarray.push("'" + kw.arg.v + "'"); - kwarray.push(this.vexpr(kw.value)); - } else { - hasStars = true; - } - } - keywordArgs = "[" + kwarray.join(",") + "]"; - if (hasStars) { - keywordArgs = this._gr("keywordArgs", keywordArgs); - for (let kw of e.keywords) { - if (!kw.arg) { - out("$ret = Sk.abstr.mappingUnpackIntoKeywordArray(",keywordArgs,",",this.vexpr(kw.value),",",func,");"); - this._checkSuspension(); - } - } - } - } - if (Sk.__future__.super_args && e.func.id && e.func.id.v === "super" && positionalArgs === "[]") { // make sure there is a self variable // note that it's part of the js API spec: https://developer.mozilla.org/en/docs/Web/API/Window/self From c2b2323a9c8fc5387c346d98882036a35751a623 Mon Sep 17 00:00:00 2001 From: stu Date: Sat, 3 Sep 2022 13:03:13 +0800 Subject: [PATCH 053/137] fix: when using tp$descr_get always send the second arg for internal calls. --- src/slotdefs.js | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/slotdefs.js b/src/slotdefs.js index f85988e4ab..96aef928c9 100644 --- a/src/slotdefs.js +++ b/src/slotdefs.js @@ -121,7 +121,7 @@ function wrapperCallBack(wrapper, callback) { */ function slotFuncNoArgs(dunderFunc) { return function () { - const func = dunderFunc.tp$descr_get ? dunderFunc.tp$descr_get(this) : dunderFunc; + const func = dunderFunc.tp$descr_get ? dunderFunc.tp$descr_get(this, this.ob$type) : dunderFunc; return Sk.misceval.callsimArray(func, []); }; } @@ -136,7 +136,7 @@ function slotFuncNoArgs(dunderFunc) { function slotFuncNoArgsWithCheck(dunderName, checkFunc, checkMsg, f) { return function (dunderFunc) { return function () { - const func = dunderFunc.tp$descr_get ? dunderFunc.tp$descr_get(this) : dunderFunc; + const func = dunderFunc.tp$descr_get ? dunderFunc.tp$descr_get(this, this.ob$type) : dunderFunc; let res = Sk.misceval.callsimArray(func, []); if (!checkFunc(res)) { throw new Sk.builtin.TypeError(dunderName + " should return " + checkMsg + " (returned " + Sk.abstr.typeName(res) + ")"); @@ -152,7 +152,7 @@ function slotFuncNoArgsWithCheck(dunderName, checkFunc, checkMsg, f) { function slotFuncOneArg(dunderFunc) { return function (value) { - const func = dunderFunc.tp$descr_get ? dunderFunc.tp$descr_get(this) : dunderFunc; + const func = dunderFunc.tp$descr_get ? dunderFunc.tp$descr_get(this, this.ob$type) : dunderFunc; return Sk.misceval.callsimArray(func, [value]); }; } @@ -164,7 +164,7 @@ function slotFuncGetAttribute(pyName, canSuspend) { return getattributeFn.d$wrapped.call(this, pyName, canSuspend); } if (getattributeFn.tp$descr_get) { - getattributeFn = getattributeFn.tp$descr_get(this); + getattributeFn = getattributeFn.tp$descr_get(this, this.ob$type); } const ret = Sk.misceval.tryCatch( () => Sk.misceval.callsimOrSuspendArray(getattributeFn, [pyName]), @@ -181,7 +181,7 @@ function slotFuncGetAttribute(pyName, canSuspend) { function slotFuncFastCall(dunderFunc) { return function (args, kwargs) { - const func = dunderFunc.tp$descr_get ? dunderFunc.tp$descr_get(this) : dunderFunc; + const func = dunderFunc.tp$descr_get ? dunderFunc.tp$descr_get(this, this.ob$type) : dunderFunc; return Sk.misceval.callsimOrSuspendArray(func, args, kwargs); }; } @@ -270,7 +270,7 @@ Sk.slots.__init__ = { $slot_name: "tp$init", $slot_func: function (dunderFunc) { return function tp$init(args, kwargs) { - const func = dunderFunc.tp$descr_get ? dunderFunc.tp$descr_get(this) : dunderFunc; + const func = dunderFunc.tp$descr_get ? dunderFunc.tp$descr_get(this, this.ob$type) : dunderFunc; let ret = Sk.misceval.callsimOrSuspendArray(func, args, kwargs); return Sk.misceval.chain(ret, (r) => { if (!Sk.builtin.checkNone(r) && r !== undefined) { @@ -441,7 +441,7 @@ slots.__getattribute__ = { return val; } if (getattrFn.tp$descr_get) { - getattrFn = getattrFn.tp$descr_get(this); + getattrFn = getattrFn.tp$descr_get(this, this.ob$type); } return Sk.misceval.callsimOrSuspendArray(getattrFn, [pyName]); }, @@ -561,7 +561,7 @@ slots.__get__ = { if (obtype == null) { obtype = Sk.builtin.none.none$; } - const func = dunderFunc.tp$descr_get ? dunderFunc.tp$descr_get(this) : dunderFunc; + const func = dunderFunc.tp$descr_get ? dunderFunc.tp$descr_get(this, this.ob$type) : dunderFunc; const ret = Sk.misceval.callsimOrSuspendArray(func, [obj, obtype]); return canSuspend ? ret : Sk.misceval.retryOptionalSuspensionOrThrow(ret); }; @@ -762,7 +762,7 @@ slots.__next__ = { $slot_name: "tp$iternext", $slot_func: function (dunderFunc) { return function tp$iternext(canSuspend) { - const func = dunderFunc.tp$descr_get ? dunderFunc.tp$descr_get(this) : dunderFunc; + const func = dunderFunc.tp$descr_get ? dunderFunc.tp$descr_get(this, this.ob$type) : dunderFunc; const ret = Sk.misceval.tryCatch( () => Sk.misceval.callsimOrSuspendArray(func, []), (e) => { @@ -850,7 +850,7 @@ slots.__len__ = { $slot_func: function (dunderFunc) { return function sq$length(canSuspend) { let res; - const func = dunderFunc.tp$descr_get ? dunderFunc.tp$descr_get(this) : dunderFunc; + const func = dunderFunc.tp$descr_get ? dunderFunc.tp$descr_get(this, this.ob$type) : dunderFunc; if (canSuspend) { res = Sk.misceval.callsimOrSuspendArray(func, []); return Sk.misceval.chain(res, (r) => { @@ -884,7 +884,7 @@ slots.__contains__ = { $slot_name: "sq$contains", $slot_func: function (dunderFunc) { return function sq$contains(key, canSuspend) { - const func = dunderFunc.tp$descr_get ? dunderFunc.tp$descr_get(this) : dunderFunc; + const func = dunderFunc.tp$descr_get ? dunderFunc.tp$descr_get(this, this.ob$type) : dunderFunc; let res = Sk.misceval.callsimOrSuspendArray(func, [key]); res = Sk.misceval.chain(res, (r) => Sk.misceval.isTrue(r)); if (res.$isSuspension) { @@ -915,7 +915,7 @@ slots.__getitem__ = { $slot_name: "mp$subscript", $slot_func: function (dunderFunc) { return function mp$subscript(key, canSuspend) { - const func = dunderFunc.tp$descr_get ? dunderFunc.tp$descr_get(this) : dunderFunc; + const func = dunderFunc.tp$descr_get ? dunderFunc.tp$descr_get(this, this.ob$type) : dunderFunc; const ret = Sk.misceval.callsimOrSuspendArray(func, [key]); return canSuspend ? ret : Sk.misceval.retryOptionalSuspensionOrThrow(ret); }; @@ -1647,7 +1647,7 @@ slots.__pow__ = { $slot_name: "nb$power", $slot_func: function (dunderFunc) { return function (value, mod) { - const func = dunderFunc.tp$descr_get ? dunderFunc.tp$descr_get(this) : dunderFunc; + const func = dunderFunc.tp$descr_get ? dunderFunc.tp$descr_get(this, this.ob$type) : dunderFunc; if (mod == undefined) { return Sk.misceval.callsimArray(func, [value]); } else { From 5327ada4a17634a7bd180331ce092158a9a9e6c1 Mon Sep 17 00:00:00 2001 From: stu Date: Fri, 19 Aug 2022 16:21:53 +0800 Subject: [PATCH 054/137] add failing test for Suspension bug with object.__getatribute__ --- test/unit3/test_suspensions.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/unit3/test_suspensions.py b/test/unit3/test_suspensions.py index f2818be635..e96454e999 100644 --- a/test/unit3/test_suspensions.py +++ b/test/unit3/test_suspensions.py @@ -31,6 +31,16 @@ def __iter__(self): sleep(0.01) return iter([0, 1, 2]) +class SleepingObjectGetatttr: + @property + def _foo(self): + sleep(.01) + return "foo" + + @property + def foo(self): + return object.__getattribute__(self, "_foo") + class Test_Suspensions(unittest.TestCase): def test_min_max(self): x = [4, 1, 5] @@ -85,6 +95,9 @@ def test_suspension_error(self): pass self.assertIn("Cannot call a function that blocks or suspends", str(e.exception)) self.assertTrue(repr(e.exception).startswith("SuspensionError")) + + def test_suspension_bug_getattribute(self): + self.assertEqual(SleepingObjectGetatttr().foo, "foo") From 21ebb49d9c31ce1eb0ea9242cb784d986bdbb49e Mon Sep 17 00:00:00 2001 From: stu Date: Fri, 19 Aug 2022 16:08:03 +0800 Subject: [PATCH 055/137] fix __getattribute__ should be suspension aware --- src/slotdefs.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/slotdefs.js b/src/slotdefs.js index 96aef928c9..b4cab55dc1 100644 --- a/src/slotdefs.js +++ b/src/slotdefs.js @@ -464,11 +464,13 @@ slots.__getattribute__ = { if (!Sk.builtin.checkString(pyName)) { throw new Sk.builtin.TypeError("attribute name must be string, not '" + Sk.abstr.typeName(pyName) + "'"); } - const res = this.call(self, pyName); - if (res === undefined) { - throw new Sk.builtin.AttributeError(Sk.abstr.typeName(self) + " has no attribute " + pyName.$jsstr()); - } - return res; + const res = this.call(self, pyName, true); + return Sk.misceval.chain(res, (res) => { + if (res === undefined) { + throw new Sk.builtin.AttributeError(Sk.abstr.typeName(self) + " has no attribute " + pyName.$jsstr()); + } + return res; + }); }, $textsig: "($self, name, /)", $flags: { OneArg: true }, From 8cd06efbf0c668568e27e562d8f0aceebbbcf476 Mon Sep 17 00:00:00 2001 From: stu Date: Fri, 19 Aug 2022 16:41:31 +0800 Subject: [PATCH 056/137] Add an additional failing test for suspension bug with `__get__` and support suspensions for valid dunders. Not used internally for Skulpt, but in theory could be used by others. --- src/slotdefs.js | 43 ++++++++++++++++++++++++++-------- test/unit3/test_suspensions.py | 4 +++- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/src/slotdefs.js b/src/slotdefs.js index b4cab55dc1..6dbcf5de56 100644 --- a/src/slotdefs.js +++ b/src/slotdefs.js @@ -24,6 +24,19 @@ function wrapperCallNoArgs(self, args, kwargs) { } return res; } + +function wrapperCallNoArgsSuspend(self, args, kwargs) { + // this = the wrapped function + Sk.abstr.checkNoArgs(this.$name, args, kwargs); + const res = this.call(self, true); + return Sk.misceval.chain(res, (res) => { + if (res === undefined) { + return Sk.builtin.none.none$; + } + return res; + }); +} + /** * @param {*} self * @param {Array} args @@ -55,6 +68,18 @@ function wrapperCallOneArg(self, args, kwargs) { return res; } +function wrapperCallOneArgSuspend(self, args, kwargs) { + // this = the wrapped function + Sk.abstr.checkOneArg(this.$name, args, kwargs); + const res = this.call(self, args[0], true); + return Sk.misceval.chain(res, (res) => { + if (res === undefined) { + return Sk.builtin.none.none$; + } + return res; + }); +} + /** * @param {*} self * @param {!Array} args @@ -81,8 +106,7 @@ function wrapperCallTernary(self, args, kwargs) { function wrapperSet(self, args, kwargs) { Sk.abstr.checkNoKwargs(this.$name, kwargs); Sk.abstr.checkArgsLen(this.$name, args, 2, 2); - this.call(self, args[0], args[1]); - return Sk.builtin.none.none$; + return Sk.misceval.chain(this.call(self, args[0], args[1], true), () => Sk.builtin.none.none$); } /** @@ -99,10 +123,10 @@ function wrapperRichCompare(self, args, kwargs) { return new Sk.builtin.bool(res); } -function wrapperCallBack(wrapper, callback) { +function wrapperCallBack(wrapper, callback, canSuspend) { return function (self,args, kwargs) { const res = wrapper.call(this, self, args, kwargs); - return callback(res); + return canSuspend ? Sk.misceval.chain(res, callback) : callback(res); }; } @@ -520,8 +544,7 @@ slots.__setattr__ = { Sk.abstr.checkNoKwargs(this.$name, kwargs); Sk.abstr.checkArgsLen(this.$name, args, 2, 2); hackcheck(self, this); - this.call(self, args[0], args[1]); - return Sk.builtin.none.none$; + return Sk.misceval.chain(this.call(self, args[0], args[1], true), () => Sk.builtin.none.none$); }, $textsig: "($self, name, value, /)", $flags: { MinArgs: 2, MaxArgs: 2 }, @@ -582,7 +605,7 @@ slots.__get__ = { if (obtype === null && obj === null) { throw new Sk.builtin.TypeError("__get__(None, None) is invalid"); } - return this.call(self, obj, obtype); + return this.call(self, obj, obtype, true); }, $textsig: "($self, instance, owner, /)", $flags: { MinArgs: 2, MaxArgs: 2 }, @@ -864,7 +887,7 @@ slots.__len__ = { } }; }, - $wrapper: wrapperCallBack(wrapperCallNoArgs, (res) => new Sk.builtin.int_(res)), + $wrapper: wrapperCallBack(wrapperCallNoArgsSuspend, (res) => new Sk.builtin.int_(res), true), $flags: { NoArgs: true }, $textsig: "($self, /)", $doc: "Return len(self).", @@ -922,7 +945,7 @@ slots.__getitem__ = { return canSuspend ? ret : Sk.misceval.retryOptionalSuspensionOrThrow(ret); }; }, - $wrapper: wrapperCallOneArg, + $wrapper: wrapperCallOneArgSuspend, $textsig: "($self, key, /)", $flags: { OneArg: true }, $doc: "Return self[key].", @@ -955,7 +978,7 @@ slots.__delitem__ = { $name: "__delitem__", $slot_name: "mp$ass_subscript", $slot_func: slots.__setitem__.$slot_func, - $wrapper: wrapperCallOneArg, + $wrapper: wrapperCallOneArgSuspend, $textsig: "($self, key, /)", $flags: { OneArg: true }, $doc: "Delete self[key].", diff --git a/test/unit3/test_suspensions.py b/test/unit3/test_suspensions.py index e96454e999..d8dafc8ccc 100644 --- a/test/unit3/test_suspensions.py +++ b/test/unit3/test_suspensions.py @@ -97,7 +97,9 @@ def test_suspension_error(self): self.assertTrue(repr(e.exception).startswith("SuspensionError")) def test_suspension_bug_getattribute(self): - self.assertEqual(SleepingObjectGetatttr().foo, "foo") + obj = SleepingObjectGetatttr() + self.assertEqual(obj.foo, "foo") + self.assertEqual(SleepingObjectGetatttr._foo.__get__(obj, None), "foo") From 15a6c817593cb25fc3997d6236586424dfe43e63 Mon Sep 17 00:00:00 2001 From: stu Date: Thu, 10 Nov 2022 21:36:27 +0800 Subject: [PATCH 057/137] Tweak implementation - have a fail safe for the non-suspension case and allow __contains__ to suspend --- src/slotdefs.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/slotdefs.js b/src/slotdefs.js index 6dbcf5de56..5c81dd7f6e 100644 --- a/src/slotdefs.js +++ b/src/slotdefs.js @@ -124,9 +124,11 @@ function wrapperRichCompare(self, args, kwargs) { } function wrapperCallBack(wrapper, callback, canSuspend) { - return function (self,args, kwargs) { + return function (self, args, kwargs) { const res = wrapper.call(this, self, args, kwargs); - return canSuspend ? Sk.misceval.chain(res, callback) : callback(res); + return canSuspend + ? Sk.misceval.chain(res, callback) + : callback(Sk.misceval.retryOptionalSuspensionOrThrow(res)); }; } @@ -918,8 +920,7 @@ slots.__contains__ = { return res; }; }, - // todo - allow for suspensions - but no internal functions suspend here - $wrapper: wrapperCallBack(wrapperCallOneArg, (res) => new Sk.builtin.bool(res)), + $wrapper: wrapperCallBack(wrapperCallOneArgSuspend, (res) => new Sk.builtin.bool(res), true), $textsig: "($self, key, /)", $flags: { OneArg: true }, $doc: "Return key in self.", From 1670ec08ac2df28d08a58f824ab6569af8d59a92 Mon Sep 17 00:00:00 2001 From: Stu Cork Date: Tue, 25 May 2021 16:38:49 +0000 Subject: [PATCH 058/137] implement uuid --- src/lib/uuid.js | 141 ++++++++++++++++++++++++++++++++++++++++++++++++ src/lib/uuid.py | 1 - 2 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 src/lib/uuid.js delete mode 100644 src/lib/uuid.py diff --git a/src/lib/uuid.js b/src/lib/uuid.js new file mode 100644 index 0000000000..4d2704249d --- /dev/null +++ b/src/lib/uuid.js @@ -0,0 +1,141 @@ +function $builtinmodule() { + const uuid = { + __name__: new pyStr("uuid"), + }; + + // Unique ID creation requires a high quality random # generator. In the browser we therefore + // require the crypto API and do not support built-in fallback to lower quality random number + // generators (like Math.random()). + + let getRandomValues; + + const rnds8 = new Uint8Array(16); + + function rng() { + // lazy load so that environments that need to polyfill have a chance to do so + if (!getRandomValues) { + // getRandomValues needs to be invoked in a context where "this" is a Crypto implementation. Also, + // find the complete implementation of crypto (msCrypto) on IE11. + getRandomValues = + (typeof crypto !== "undefined" && crypto.getRandomValues && crypto.getRandomValues.bind(crypto)) || + (typeof msCrypto !== "undefined" && + typeof msCrypto.getRandomValues === "function" && + msCrypto.getRandomValues.bind(msCrypto)); + } + return getRandomValues(rnds8); + } + + const byteToHex = []; + + for (let i = 0; i < 256; ++i) { + byteToHex.push((i + 0x100).toString(16).substr(1)); + } + + function stringify(arr, offset = 0) { + // Note: Be careful editing this code! It's been tuned for performance + // and works in ways you may not expect. See https://github.com/uuidjs/uuid/pull/434 + const uuid = ( + byteToHex[arr[offset + 0]] + + byteToHex[arr[offset + 1]] + + byteToHex[arr[offset + 2]] + + byteToHex[arr[offset + 3]] + + "-" + + byteToHex[arr[offset + 4]] + + byteToHex[arr[offset + 5]] + + "-" + + byteToHex[arr[offset + 6]] + + byteToHex[arr[offset + 7]] + + "-" + + byteToHex[arr[offset + 8]] + + byteToHex[arr[offset + 9]] + + "-" + + byteToHex[arr[offset + 10]] + + byteToHex[arr[offset + 11]] + + byteToHex[arr[offset + 12]] + + byteToHex[arr[offset + 13]] + + byteToHex[arr[offset + 14]] + + byteToHex[arr[offset + 15]] + ).toLowerCase(); + + // Consistency check for valid UUID. If this throws, it's likely due to one + // of the following: + // - One or more input array values don't map to a hex octet (leading to + // "undefined" in the uuid) + // - Invalid input values for the RFC `version` or `variant` fields + if (!validate(uuid)) { + throw TypeError("Stringified UUID is invalid"); + } + + return uuid; + } + + function v35(name, version, hashfunc) { + function generateUUID(value, namespace, buf, offset) { + if (typeof value === "string") { + value = stringToBytes(value); + } + + if (typeof namespace === "string") { + namespace = parse(namespace); + } + + if (namespace.length !== 16) { + throw TypeError("Namespace must be array-like (16 iterable integer values, 0-255)"); + } + + // Compute hash of namespace and value, Per 4.3 + // Future: Use spread syntax when supported on all platforms, e.g. `bytes = + // hashfunc([...namespace, ... value])` + let bytes = new Uint8Array(16 + value.length); + bytes.set(namespace); + bytes.set(value, namespace.length); + bytes = hashfunc(bytes); + + bytes[6] = (bytes[6] & 0x0f) | version; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + + return bytes; + } + // Function#name is not settable on some platforms (#270) + try { + generateUUID.name = name; + // eslint-disable-next-line no-empty + } catch (err) {} + return generateUUID; + } + + import v35 from './v35.js'; +import sha1 from './sha1.js'; + +const v5 = v35('v5', 0x50, sha1); + + function v4() { + const rnds = rnt(); + // Per 4.4, set bits for version and `clock_seq_hi_and_reserved` + rnds[6] = (rnds[6] & 0x0f) | 0x40; + rnds[8] = (rnds[8] & 0x3f) | 0x80; + } + + uuid.UUID = buildNativeClass("uuid.UUID", { + constructor: function UUID() {}, + slots: { + tp$str() { + return new pyStr(stringify(this.$b)); + }, + }, + getsets: { + bytes: {}, + int: {}, + hex: {}, + }, + }); + + setUpModuleMethods("uuid", uuid, { + uuid4: { + $meth() { + const bytes = v4(); + return new uuid.UUID(bytes); + }, + }, + }); +} diff --git a/src/lib/uuid.py b/src/lib/uuid.py deleted file mode 100644 index ea7c06bb64..0000000000 --- a/src/lib/uuid.py +++ /dev/null @@ -1 +0,0 @@ -import _sk_fail; _sk_fail._("uuid") From b8277d0eab835923eae870ebcd2c5a61453fa29d Mon Sep 17 00:00:00 2001 From: stu Date: Tue, 8 Nov 2022 14:54:05 +0800 Subject: [PATCH 059/137] wip uuid --- src/lib/uuid.js | 360 ++++++++++++++++++++++++++++++++--------------- src/lib/uuid_.js | 141 +++++++++++++++++++ 2 files changed, 386 insertions(+), 115 deletions(-) create mode 100644 src/lib/uuid_.js diff --git a/src/lib/uuid.js b/src/lib/uuid.js index 4d2704249d..fe4fd2c40d 100644 --- a/src/lib/uuid.js +++ b/src/lib/uuid.js @@ -1,141 +1,271 @@ function $builtinmodule() { - const uuid = { + const { + builtin: { + bytes: pyBytes, + str: pyStr, + int: pyInt, + TypeError: pyTypeError, + ValueError: pyValueError, + NotImplementedError: pyNotImplementedError, + none: { none$: pyNone }, + NotImplemented: { NotImplemented$: pyNotImplemented }, + len: pyLen, + }, + abstr: { buildNativeClass, checkArgsLen, copyKeywordsToNamedArgs }, + misceval: { pyCall, richCompareBool }, + } = Sk; + + const mod = { __name__: new pyStr("uuid"), }; - // Unique ID creation requires a high quality random # generator. In the browser we therefore - // require the crypto API and do not support built-in fallback to lower quality random number - // generators (like Math.random()). - - let getRandomValues; - - const rnds8 = new Uint8Array(16); - - function rng() { - // lazy load so that environments that need to polyfill have a chance to do so - if (!getRandomValues) { - // getRandomValues needs to be invoked in a context where "this" is a Crypto implementation. Also, - // find the complete implementation of crypto (msCrypto) on IE11. - getRandomValues = - (typeof crypto !== "undefined" && crypto.getRandomValues && crypto.getRandomValues.bind(crypto)) || - (typeof msCrypto !== "undefined" && - typeof msCrypto.getRandomValues === "function" && - msCrypto.getRandomValues.bind(msCrypto)); - } - return getRandomValues(rnds8); - } - - const byteToHex = []; + const fromBytes = pyInt.tp$getattr(new pyStr("from_bytes")); + const toBytes = pyInt.tp$getattr(new pyStr("to_bytes")); + const _intMax = new pyInt(1).nb$lshift(new pyInt(128)); + const _0 = new pyInt(0); + const _4 = new pyInt(4); + const _16 = new pyInt(16); - for (let i = 0; i < 256; ++i) { - byteToHex.push((i + 0x100).toString(16).substr(1)); - } - - function stringify(arr, offset = 0) { - // Note: Be careful editing this code! It's been tuned for performance - // and works in ways you may not expect. See https://github.com/uuidjs/uuid/pull/434 - const uuid = ( - byteToHex[arr[offset + 0]] + - byteToHex[arr[offset + 1]] + - byteToHex[arr[offset + 2]] + - byteToHex[arr[offset + 3]] + - "-" + - byteToHex[arr[offset + 4]] + - byteToHex[arr[offset + 5]] + - "-" + - byteToHex[arr[offset + 6]] + - byteToHex[arr[offset + 7]] + - "-" + - byteToHex[arr[offset + 8]] + - byteToHex[arr[offset + 9]] + - "-" + - byteToHex[arr[offset + 10]] + - byteToHex[arr[offset + 11]] + - byteToHex[arr[offset + 12]] + - byteToHex[arr[offset + 13]] + - byteToHex[arr[offset + 14]] + - byteToHex[arr[offset + 15]] - ).toLowerCase(); + const _s_big = new pyStr("big"); + const _s_32bit = new pyStr("%032x"); - // Consistency check for valid UUID. If this throws, it's likely due to one - // of the following: - // - One or more input array values don't map to a hex octet (leading to - // "undefined" in the uuid) - // - Invalid input values for the RFC `version` or `variant` fields - if (!validate(uuid)) { - throw TypeError("Stringified UUID is invalid"); - } + const lt = (a, b) => richCompareBool(a, b, "Lt"); + const ge = (a, b) => richCompareBool(a, b, "GtE"); - return uuid; + function notImplemneted() { + throw new pyNotImplementedError("Not yet implemneted in Skulpt"); } - function v35(name, version, hashfunc) { - function generateUUID(value, namespace, buf, offset) { - if (typeof value === "string") { - value = stringToBytes(value); - } - - if (typeof namespace === "string") { - namespace = parse(namespace); - } - - if (namespace.length !== 16) { - throw TypeError("Namespace must be array-like (16 iterable integer values, 0-255)"); - } - - // Compute hash of namespace and value, Per 4.3 - // Future: Use spread syntax when supported on all platforms, e.g. `bytes = - // hashfunc([...namespace, ... value])` - let bytes = new Uint8Array(16 + value.length); - bytes.set(namespace); - bytes.set(value, namespace.length); - bytes = hashfunc(bytes); - - bytes[6] = (bytes[6] & 0x0f) | version; - bytes[8] = (bytes[8] & 0x3f) | 0x80; + const UUID = (mod.UUID = buildNativeClass("uuid.UUID", { + constructor: function () {}, + slots: { + tp$init(args, kws) { + checkArgsLen("UUID", args, 0, 6); + let [hex, bytes, bytes_le, fields, int, version, is_safe] = copyKeywordsToNamedArgs( + "UUID", + ["hex", "bytes", "bytes_le", "fields", , "version", "is_safe"], + args, + kws, + [pyNone, pyNone, pyNone, pyNone, pyNone, pyNone, pyNone] + ); - return bytes; - } - // Function#name is not settable on some platforms (#270) - try { - generateUUID.name = name; - // eslint-disable-next-line no-empty - } catch (err) {} - return generateUUID; - } + if ([hex, bytes, bytes_le, fields, int].filter((x) => x === pyNone).length !== 4) { + throw new pyTypeError("one of the hex, bytes, bytes_le, fields, or int arguments must be given"); + } - import v35 from './v35.js'; -import sha1 from './sha1.js'; + if (hex !== pyNone) { + hex = hex.toString().replace("urn:", "").replace("uuid:", ""); + let start = 0, + end = hex.length - 1; + while ("{}".indexOf(hex[start] >= 0)) { + start++; + } + while ("{}".indexOf(hex[end] >= 0)) { + end--; + } + hex = hex.slice(start, end + 1); + hex = hex.replaceAll("-", ""); + if (hex.length !== 32) { + throw new pyValueError("badly formed hexadecimal UUID string"); + } + bytes = [ + bytes_le[3], + bytes_le[2], + bytes_le[1], + bytes_le[0], + bytes_le[5], + bytes_le[4], + bytes_le[7], + bytes_le[6], + ]; + bytes.push(...bytes_le.slice(8)); + bytes = new pyBytes(bytes); + } -const v5 = v35('v5', 0x50, sha1); + if (bytes_le !== pyNone) { + if (!(bytes_le instanceof pyBytes)) { + throw new pyTypeError("bytes_le should be a bytes instance"); + } + bytes_le = bytes_le.valueOf(); + if (bytes_le.length !== 16) { + throw new pyValueError("bytes_le is not a 16-char string"); + } + bytes = new pyBytes(); + } + if (bytes !== pyNone) { + if (!(bytes instanceof pyBytes)) { + throw new pyTypeError("bytes_le should be a bytes instance"); + } + if (!bytes.valueOf().length !== 16) { + throw new pyValueError("bytes is not a 16-char string"); + } + int = pyCall(fromBytes, [bytes], ["byteorder", _s_big]); + } - function v4() { - const rnds = rnt(); - // Per 4.4, set bits for version and `clock_seq_hi_and_reserved` - rnds[6] = (rnds[6] & 0x0f) | 0x40; - rnds[8] = (rnds[8] & 0x3f) | 0x80; - } + if (int !== pyNone) { + if (lt(int, _0) || ge(int, _intMax)) { + throw new pyValueError("int is out of range (need a 128-bit value)"); + } + } - uuid.UUID = buildNativeClass("uuid.UUID", { - constructor: function UUID() {}, - slots: { + this.$int = int; + this.$isSafe = is_safe; + }, tp$str() { - return new pyStr(stringify(this.$b)); + const hex = _s_32bit.nb$remainder(this.$int).toString(); + return new pyStr( + `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}` + ); + }, + tp$repr() { + const name = this.tp$name; + const s = new pyStr(this); + return new pyStr(`${name}(${s})`); + }, + tp$hash() { + return this.$int.tp$hash(); + }, + tp$richcompare(other, op) { + if (!(other instanceof UUID)) { + return pyNotImplemented; + } + return this.$int.tp$richcompare(other, op); }, }, + methods: { + __int__: { + $meth() { + return this.$int; + }, + $flags: { NoArgs: true }, + }, + }, + getsets: { - bytes: {}, - int: {}, - hex: {}, + int: { + $get() { + return this.$int; + }, + }, + is_safe: { + $get() { + return this.$isSafe; + }, + }, + bytes: { + $get() { + return pyCall(toBytes, [this.$int, _16, _s_big]); + }, + }, + bytes_le: { + $get() { + const bytes = this.tp$getattr(new pyStr("bytes")).valueOf(); + const bytes_le = [bytes[3], bytes[2], bytes[1], bytes[0], bytes[5], bytes[4], bytes[7], bytes[6]]; + bytes_le.push(...bytes.slice(8)); + return new pyBytes(bytes_le); + }, + }, + fields: { + $get() { + return notImplemneted(); + }, + }, + time_low: { + $get() { + return notImplemneted(); + }, + }, + time_mid: { + $get() { + return notImplemneted(); + }, + }, + time_hi_version: { + $get() { + return notImplemneted(); + }, + }, + clock_seq_hi_variant: { + $get() { + return notImplemneted(); + }, + }, + clock_seq_low: { + $get() { + return notImplemneted(); + }, + }, + time: { + $get() { + return notImplemneted(); + }, + }, + clock_seq: { + $get() { + return notImplemneted(); + }, + }, + node: { + $get() { + return notImplemneted(); + }, + }, + hex: { + $get() { + return notImplemneted(); + }, + }, + urn: { + $get() { + return notImplemneted(); + }, + }, + variant: { + $get() { + return notImplemneted(); + }, + }, + version: { + $get() { + return notImplemneted(); + }, + }, }, - }); + })); - setUpModuleMethods("uuid", uuid, { + setUpModuleMethods("uuid", mod, { + uuid1: { + $meth() { + notImplemneted(); + }, + $flags: { FastCall: true }, + }, + uuid2: { + $meth() { + notImplemneted(); + }, + $flags: { FastCall: true }, + }, + uuid3: { + $meth() { + notImplemneted(); + }, + $flags: { FastCall: true }, + }, uuid4: { $meth() { - const bytes = v4(); - return new uuid.UUID(bytes); + const bytes = new pyBytes(crypto.getRandomValues(new Uint8Array(16))); + return pyCall(UUID, [], ["bytes", bytes, "version", _4]); }, + $flags: { NoArgs: true }, + }, + uuid4: { + $meth() { + notImplemneted(); + }, + $flags: { FastCall: true }, }, }); + + return mod; } diff --git a/src/lib/uuid_.js b/src/lib/uuid_.js new file mode 100644 index 0000000000..4d2704249d --- /dev/null +++ b/src/lib/uuid_.js @@ -0,0 +1,141 @@ +function $builtinmodule() { + const uuid = { + __name__: new pyStr("uuid"), + }; + + // Unique ID creation requires a high quality random # generator. In the browser we therefore + // require the crypto API and do not support built-in fallback to lower quality random number + // generators (like Math.random()). + + let getRandomValues; + + const rnds8 = new Uint8Array(16); + + function rng() { + // lazy load so that environments that need to polyfill have a chance to do so + if (!getRandomValues) { + // getRandomValues needs to be invoked in a context where "this" is a Crypto implementation. Also, + // find the complete implementation of crypto (msCrypto) on IE11. + getRandomValues = + (typeof crypto !== "undefined" && crypto.getRandomValues && crypto.getRandomValues.bind(crypto)) || + (typeof msCrypto !== "undefined" && + typeof msCrypto.getRandomValues === "function" && + msCrypto.getRandomValues.bind(msCrypto)); + } + return getRandomValues(rnds8); + } + + const byteToHex = []; + + for (let i = 0; i < 256; ++i) { + byteToHex.push((i + 0x100).toString(16).substr(1)); + } + + function stringify(arr, offset = 0) { + // Note: Be careful editing this code! It's been tuned for performance + // and works in ways you may not expect. See https://github.com/uuidjs/uuid/pull/434 + const uuid = ( + byteToHex[arr[offset + 0]] + + byteToHex[arr[offset + 1]] + + byteToHex[arr[offset + 2]] + + byteToHex[arr[offset + 3]] + + "-" + + byteToHex[arr[offset + 4]] + + byteToHex[arr[offset + 5]] + + "-" + + byteToHex[arr[offset + 6]] + + byteToHex[arr[offset + 7]] + + "-" + + byteToHex[arr[offset + 8]] + + byteToHex[arr[offset + 9]] + + "-" + + byteToHex[arr[offset + 10]] + + byteToHex[arr[offset + 11]] + + byteToHex[arr[offset + 12]] + + byteToHex[arr[offset + 13]] + + byteToHex[arr[offset + 14]] + + byteToHex[arr[offset + 15]] + ).toLowerCase(); + + // Consistency check for valid UUID. If this throws, it's likely due to one + // of the following: + // - One or more input array values don't map to a hex octet (leading to + // "undefined" in the uuid) + // - Invalid input values for the RFC `version` or `variant` fields + if (!validate(uuid)) { + throw TypeError("Stringified UUID is invalid"); + } + + return uuid; + } + + function v35(name, version, hashfunc) { + function generateUUID(value, namespace, buf, offset) { + if (typeof value === "string") { + value = stringToBytes(value); + } + + if (typeof namespace === "string") { + namespace = parse(namespace); + } + + if (namespace.length !== 16) { + throw TypeError("Namespace must be array-like (16 iterable integer values, 0-255)"); + } + + // Compute hash of namespace and value, Per 4.3 + // Future: Use spread syntax when supported on all platforms, e.g. `bytes = + // hashfunc([...namespace, ... value])` + let bytes = new Uint8Array(16 + value.length); + bytes.set(namespace); + bytes.set(value, namespace.length); + bytes = hashfunc(bytes); + + bytes[6] = (bytes[6] & 0x0f) | version; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + + return bytes; + } + // Function#name is not settable on some platforms (#270) + try { + generateUUID.name = name; + // eslint-disable-next-line no-empty + } catch (err) {} + return generateUUID; + } + + import v35 from './v35.js'; +import sha1 from './sha1.js'; + +const v5 = v35('v5', 0x50, sha1); + + function v4() { + const rnds = rnt(); + // Per 4.4, set bits for version and `clock_seq_hi_and_reserved` + rnds[6] = (rnds[6] & 0x0f) | 0x40; + rnds[8] = (rnds[8] & 0x3f) | 0x80; + } + + uuid.UUID = buildNativeClass("uuid.UUID", { + constructor: function UUID() {}, + slots: { + tp$str() { + return new pyStr(stringify(this.$b)); + }, + }, + getsets: { + bytes: {}, + int: {}, + hex: {}, + }, + }); + + setUpModuleMethods("uuid", uuid, { + uuid4: { + $meth() { + const bytes = v4(); + return new uuid.UUID(bytes); + }, + }, + }); +} From 4a43220f06d3ac2be0dc46e4b491f851eefcacd9 Mon Sep 17 00:00:00 2001 From: stu Date: Tue, 8 Nov 2022 16:35:51 +0800 Subject: [PATCH 060/137] Tests: add uuid tests direct from cpython --- test/unit3/test_uuid.py | 890 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 890 insertions(+) create mode 100644 test/unit3/test_uuid.py diff --git a/test/unit3/test_uuid.py b/test/unit3/test_uuid.py new file mode 100644 index 0000000000..f5a13d2255 --- /dev/null +++ b/test/unit3/test_uuid.py @@ -0,0 +1,890 @@ +import unittest +from test import support +from test.support import import_helper +import builtins +import contextlib +import copy +import io +import os +import pickle +import sys +import weakref +from unittest import mock + +py_uuid = import_helper.import_fresh_module('uuid', blocked=['_uuid']) +c_uuid = import_helper.import_fresh_module('uuid', fresh=['_uuid']) + +def importable(name): + try: + __import__(name) + return True + except: + return False + + +def mock_get_command_stdout(data): + def get_command_stdout(command, args): + return io.BytesIO(data.encode()) + return get_command_stdout + + +class BaseTestUUID: + uuid = None + + def test_UUID(self): + equal = self.assertEqual + ascending = [] + for (string, curly, hex, bytes, bytes_le, fields, integer, urn, time, clock_seq, variant, version) in [ + ('00000000-0000-0000-0000-000000000000', + '{00000000-0000-0000-0000-000000000000}', + '00000000000000000000000000000000', + b'\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0', + b'\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0', + (0, 0, 0, 0, 0, 0), + 0, + 'urn:uuid:00000000-0000-0000-0000-000000000000', + 0, 0, self.uuid.RESERVED_NCS, None), + ('00010203-0405-0607-0809-0a0b0c0d0e0f', + '{00010203-0405-0607-0809-0a0b0c0d0e0f}', + '000102030405060708090a0b0c0d0e0f', + b'\0\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\x0d\x0e\x0f', + b'\x03\x02\x01\0\x05\x04\x07\x06\x08\t\n\x0b\x0c\x0d\x0e\x0f', + (0x00010203, 0x0405, 0x0607, 8, 9, 0x0a0b0c0d0e0f), + 0x000102030405060708090a0b0c0d0e0f, + 'urn:uuid:00010203-0405-0607-0809-0a0b0c0d0e0f', + 0x607040500010203, 0x809, self.uuid.RESERVED_NCS, None), + ('02d9e6d5-9467-382e-8f9b-9300a64ac3cd', + '{02d9e6d5-9467-382e-8f9b-9300a64ac3cd}', + '02d9e6d59467382e8f9b9300a64ac3cd', + b'\x02\xd9\xe6\xd5\x94\x67\x38\x2e\x8f\x9b\x93\x00\xa6\x4a\xc3\xcd', + b'\xd5\xe6\xd9\x02\x67\x94\x2e\x38\x8f\x9b\x93\x00\xa6\x4a\xc3\xcd', + (0x02d9e6d5, 0x9467, 0x382e, 0x8f, 0x9b, 0x9300a64ac3cd), + 0x02d9e6d59467382e8f9b9300a64ac3cd, + 'urn:uuid:02d9e6d5-9467-382e-8f9b-9300a64ac3cd', + 0x82e946702d9e6d5, 0xf9b, self.uuid.RFC_4122, 3), + ('12345678-1234-5678-1234-567812345678', + '{12345678-1234-5678-1234-567812345678}', + '12345678123456781234567812345678', + b'\x12\x34\x56\x78'*4, + b'\x78\x56\x34\x12\x34\x12\x78\x56\x12\x34\x56\x78\x12\x34\x56\x78', + (0x12345678, 0x1234, 0x5678, 0x12, 0x34, 0x567812345678), + 0x12345678123456781234567812345678, + 'urn:uuid:12345678-1234-5678-1234-567812345678', + 0x678123412345678, 0x1234, self.uuid.RESERVED_NCS, None), + ('6ba7b810-9dad-11d1-80b4-00c04fd430c8', + '{6ba7b810-9dad-11d1-80b4-00c04fd430c8}', + '6ba7b8109dad11d180b400c04fd430c8', + b'\x6b\xa7\xb8\x10\x9d\xad\x11\xd1\x80\xb4\x00\xc0\x4f\xd4\x30\xc8', + b'\x10\xb8\xa7\x6b\xad\x9d\xd1\x11\x80\xb4\x00\xc0\x4f\xd4\x30\xc8', + (0x6ba7b810, 0x9dad, 0x11d1, 0x80, 0xb4, 0x00c04fd430c8), + 0x6ba7b8109dad11d180b400c04fd430c8, + 'urn:uuid:6ba7b810-9dad-11d1-80b4-00c04fd430c8', + 0x1d19dad6ba7b810, 0xb4, self.uuid.RFC_4122, 1), + ('6ba7b811-9dad-11d1-80b4-00c04fd430c8', + '{6ba7b811-9dad-11d1-80b4-00c04fd430c8}', + '6ba7b8119dad11d180b400c04fd430c8', + b'\x6b\xa7\xb8\x11\x9d\xad\x11\xd1\x80\xb4\x00\xc0\x4f\xd4\x30\xc8', + b'\x11\xb8\xa7\x6b\xad\x9d\xd1\x11\x80\xb4\x00\xc0\x4f\xd4\x30\xc8', + (0x6ba7b811, 0x9dad, 0x11d1, 0x80, 0xb4, 0x00c04fd430c8), + 0x6ba7b8119dad11d180b400c04fd430c8, + 'urn:uuid:6ba7b811-9dad-11d1-80b4-00c04fd430c8', + 0x1d19dad6ba7b811, 0xb4, self.uuid.RFC_4122, 1), + ('6ba7b812-9dad-11d1-80b4-00c04fd430c8', + '{6ba7b812-9dad-11d1-80b4-00c04fd430c8}', + '6ba7b8129dad11d180b400c04fd430c8', + b'\x6b\xa7\xb8\x12\x9d\xad\x11\xd1\x80\xb4\x00\xc0\x4f\xd4\x30\xc8', + b'\x12\xb8\xa7\x6b\xad\x9d\xd1\x11\x80\xb4\x00\xc0\x4f\xd4\x30\xc8', + (0x6ba7b812, 0x9dad, 0x11d1, 0x80, 0xb4, 0x00c04fd430c8), + 0x6ba7b8129dad11d180b400c04fd430c8, + 'urn:uuid:6ba7b812-9dad-11d1-80b4-00c04fd430c8', + 0x1d19dad6ba7b812, 0xb4, self.uuid.RFC_4122, 1), + ('6ba7b814-9dad-11d1-80b4-00c04fd430c8', + '{6ba7b814-9dad-11d1-80b4-00c04fd430c8}', + '6ba7b8149dad11d180b400c04fd430c8', + b'\x6b\xa7\xb8\x14\x9d\xad\x11\xd1\x80\xb4\x00\xc0\x4f\xd4\x30\xc8', + b'\x14\xb8\xa7\x6b\xad\x9d\xd1\x11\x80\xb4\x00\xc0\x4f\xd4\x30\xc8', + (0x6ba7b814, 0x9dad, 0x11d1, 0x80, 0xb4, 0x00c04fd430c8), + 0x6ba7b8149dad11d180b400c04fd430c8, + 'urn:uuid:6ba7b814-9dad-11d1-80b4-00c04fd430c8', + 0x1d19dad6ba7b814, 0xb4, self.uuid.RFC_4122, 1), + ('7d444840-9dc0-11d1-b245-5ffdce74fad2', + '{7d444840-9dc0-11d1-b245-5ffdce74fad2}', + '7d4448409dc011d1b2455ffdce74fad2', + b'\x7d\x44\x48\x40\x9d\xc0\x11\xd1\xb2\x45\x5f\xfd\xce\x74\xfa\xd2', + b'\x40\x48\x44\x7d\xc0\x9d\xd1\x11\xb2\x45\x5f\xfd\xce\x74\xfa\xd2', + (0x7d444840, 0x9dc0, 0x11d1, 0xb2, 0x45, 0x5ffdce74fad2), + 0x7d4448409dc011d1b2455ffdce74fad2, + 'urn:uuid:7d444840-9dc0-11d1-b245-5ffdce74fad2', + 0x1d19dc07d444840, 0x3245, self.uuid.RFC_4122, 1), + ('e902893a-9d22-3c7e-a7b8-d6e313b71d9f', + '{e902893a-9d22-3c7e-a7b8-d6e313b71d9f}', + 'e902893a9d223c7ea7b8d6e313b71d9f', + b'\xe9\x02\x89\x3a\x9d\x22\x3c\x7e\xa7\xb8\xd6\xe3\x13\xb7\x1d\x9f', + b'\x3a\x89\x02\xe9\x22\x9d\x7e\x3c\xa7\xb8\xd6\xe3\x13\xb7\x1d\x9f', + (0xe902893a, 0x9d22, 0x3c7e, 0xa7, 0xb8, 0xd6e313b71d9f), + 0xe902893a9d223c7ea7b8d6e313b71d9f, + 'urn:uuid:e902893a-9d22-3c7e-a7b8-d6e313b71d9f', + 0xc7e9d22e902893a, 0x27b8, self.uuid.RFC_4122, 3), + ('eb424026-6f54-4ef8-a4d0-bb658a1fc6cf', + '{eb424026-6f54-4ef8-a4d0-bb658a1fc6cf}', + 'eb4240266f544ef8a4d0bb658a1fc6cf', + b'\xeb\x42\x40\x26\x6f\x54\x4e\xf8\xa4\xd0\xbb\x65\x8a\x1f\xc6\xcf', + b'\x26\x40\x42\xeb\x54\x6f\xf8\x4e\xa4\xd0\xbb\x65\x8a\x1f\xc6\xcf', + (0xeb424026, 0x6f54, 0x4ef8, 0xa4, 0xd0, 0xbb658a1fc6cf), + 0xeb4240266f544ef8a4d0bb658a1fc6cf, + 'urn:uuid:eb424026-6f54-4ef8-a4d0-bb658a1fc6cf', + 0xef86f54eb424026, 0x24d0, self.uuid.RFC_4122, 4), + ('f81d4fae-7dec-11d0-a765-00a0c91e6bf6', + '{f81d4fae-7dec-11d0-a765-00a0c91e6bf6}', + 'f81d4fae7dec11d0a76500a0c91e6bf6', + b'\xf8\x1d\x4f\xae\x7d\xec\x11\xd0\xa7\x65\x00\xa0\xc9\x1e\x6b\xf6', + b'\xae\x4f\x1d\xf8\xec\x7d\xd0\x11\xa7\x65\x00\xa0\xc9\x1e\x6b\xf6', + (0xf81d4fae, 0x7dec, 0x11d0, 0xa7, 0x65, 0x00a0c91e6bf6), + 0xf81d4fae7dec11d0a76500a0c91e6bf6, + 'urn:uuid:f81d4fae-7dec-11d0-a765-00a0c91e6bf6', + 0x1d07decf81d4fae, 0x2765, self.uuid.RFC_4122, 1), + ('fffefdfc-fffe-fffe-fffe-fffefdfcfbfa', + '{fffefdfc-fffe-fffe-fffe-fffefdfcfbfa}', + 'fffefdfcfffefffefffefffefdfcfbfa', + b'\xff\xfe\xfd\xfc\xff\xfe\xff\xfe\xff\xfe\xff\xfe\xfd\xfc\xfb\xfa', + b'\xfc\xfd\xfe\xff\xfe\xff\xfe\xff\xff\xfe\xff\xfe\xfd\xfc\xfb\xfa', + (0xfffefdfc, 0xfffe, 0xfffe, 0xff, 0xfe, 0xfffefdfcfbfa), + 0xfffefdfcfffefffefffefffefdfcfbfa, + 'urn:uuid:fffefdfc-fffe-fffe-fffe-fffefdfcfbfa', + 0xffefffefffefdfc, 0x3ffe, self.uuid.RESERVED_FUTURE, None), + ('ffffffff-ffff-ffff-ffff-ffffffffffff', + '{ffffffff-ffff-ffff-ffff-ffffffffffff}', + 'ffffffffffffffffffffffffffffffff', + b'\xff'*16, + b'\xff'*16, + (0xffffffff, 0xffff, 0xffff, 0xff, 0xff, 0xffffffffffff), + 0xffffffffffffffffffffffffffffffff, + 'urn:uuid:ffffffff-ffff-ffff-ffff-ffffffffffff', + 0xfffffffffffffff, 0x3fff, self.uuid.RESERVED_FUTURE, None), + ]: + equivalents = [] + # Construct each UUID in several different ways. + for u in [self.uuid.UUID(string), self.uuid.UUID(curly), self.uuid.UUID(hex), + self.uuid.UUID(bytes=bytes), self.uuid.UUID(bytes_le=bytes_le), + self.uuid.UUID(fields=fields), self.uuid.UUID(int=integer), + self.uuid.UUID(urn)]: + # Test all conversions and properties of the UUID object. + equal(str(u), string) + equal(int(u), integer) + equal(u.bytes, bytes) + equal(u.bytes_le, bytes_le) + equal(u.fields, fields) + equal(u.time_low, fields[0]) + equal(u.time_mid, fields[1]) + equal(u.time_hi_version, fields[2]) + equal(u.clock_seq_hi_variant, fields[3]) + equal(u.clock_seq_low, fields[4]) + equal(u.node, fields[5]) + equal(u.hex, hex) + equal(u.int, integer) + equal(u.urn, urn) + equal(u.time, time) + equal(u.clock_seq, clock_seq) + equal(u.variant, variant) + equal(u.version, version) + equivalents.append(u) + + # Different construction methods should give the same UUID. + for u in equivalents: + for v in equivalents: + equal(u, v) + + # Bug 7380: "bytes" and "bytes_le" should give the same type. + equal(type(u.bytes), builtins.bytes) + equal(type(u.bytes_le), builtins.bytes) + + ascending.append(u) + + # Test comparison of UUIDs. + for i in range(len(ascending)): + for j in range(len(ascending)): + equal(i < j, ascending[i] < ascending[j]) + equal(i <= j, ascending[i] <= ascending[j]) + equal(i == j, ascending[i] == ascending[j]) + equal(i > j, ascending[i] > ascending[j]) + equal(i >= j, ascending[i] >= ascending[j]) + equal(i != j, ascending[i] != ascending[j]) + + # Test sorting of UUIDs (above list is in ascending order). + resorted = ascending[:] + resorted.reverse() + resorted.sort() + equal(ascending, resorted) + + def test_exceptions(self): + badvalue = lambda f: self.assertRaises(ValueError, f) + badtype = lambda f: self.assertRaises(TypeError, f) + + # Badly formed hex strings. + badvalue(lambda: self.uuid.UUID('')) + badvalue(lambda: self.uuid.UUID('abc')) + badvalue(lambda: self.uuid.UUID('1234567812345678123456781234567')) + badvalue(lambda: self.uuid.UUID('123456781234567812345678123456789')) + badvalue(lambda: self.uuid.UUID('123456781234567812345678z2345678')) + + # Badly formed bytes. + badvalue(lambda: self.uuid.UUID(bytes='abc')) + badvalue(lambda: self.uuid.UUID(bytes='\0'*15)) + badvalue(lambda: self.uuid.UUID(bytes='\0'*17)) + + # Badly formed bytes_le. + badvalue(lambda: self.uuid.UUID(bytes_le='abc')) + badvalue(lambda: self.uuid.UUID(bytes_le='\0'*15)) + badvalue(lambda: self.uuid.UUID(bytes_le='\0'*17)) + + # Badly formed fields. + badvalue(lambda: self.uuid.UUID(fields=(1,))) + badvalue(lambda: self.uuid.UUID(fields=(1, 2, 3, 4, 5))) + badvalue(lambda: self.uuid.UUID(fields=(1, 2, 3, 4, 5, 6, 7))) + + # Field values out of range. + badvalue(lambda: self.uuid.UUID(fields=(-1, 0, 0, 0, 0, 0))) + badvalue(lambda: self.uuid.UUID(fields=(0x100000000, 0, 0, 0, 0, 0))) + badvalue(lambda: self.uuid.UUID(fields=(0, -1, 0, 0, 0, 0))) + badvalue(lambda: self.uuid.UUID(fields=(0, 0x10000, 0, 0, 0, 0))) + badvalue(lambda: self.uuid.UUID(fields=(0, 0, -1, 0, 0, 0))) + badvalue(lambda: self.uuid.UUID(fields=(0, 0, 0x10000, 0, 0, 0))) + badvalue(lambda: self.uuid.UUID(fields=(0, 0, 0, -1, 0, 0))) + badvalue(lambda: self.uuid.UUID(fields=(0, 0, 0, 0x100, 0, 0))) + badvalue(lambda: self.uuid.UUID(fields=(0, 0, 0, 0, -1, 0))) + badvalue(lambda: self.uuid.UUID(fields=(0, 0, 0, 0, 0x100, 0))) + badvalue(lambda: self.uuid.UUID(fields=(0, 0, 0, 0, 0, -1))) + badvalue(lambda: self.uuid.UUID(fields=(0, 0, 0, 0, 0, 0x1000000000000))) + + # Version number out of range. + badvalue(lambda: self.uuid.UUID('00'*16, version=0)) + badvalue(lambda: self.uuid.UUID('00'*16, version=6)) + + # Integer value out of range. + badvalue(lambda: self.uuid.UUID(int=-1)) + badvalue(lambda: self.uuid.UUID(int=1<<128)) + + # Must supply exactly one of hex, bytes, fields, int. + h, b, f, i = '00'*16, b'\0'*16, (0, 0, 0, 0, 0, 0), 0 + self.uuid.UUID(h) + self.uuid.UUID(hex=h) + self.uuid.UUID(bytes=b) + self.uuid.UUID(bytes_le=b) + self.uuid.UUID(fields=f) + self.uuid.UUID(int=i) + + # Wrong number of arguments (positional). + badtype(lambda: self.uuid.UUID()) + badtype(lambda: self.uuid.UUID(h, b)) + badtype(lambda: self.uuid.UUID(h, b, b)) + badtype(lambda: self.uuid.UUID(h, b, b, f)) + badtype(lambda: self.uuid.UUID(h, b, b, f, i)) + + # Duplicate arguments. + for hh in [[], [('hex', h)]]: + for bb in [[], [('bytes', b)]]: + for bble in [[], [('bytes_le', b)]]: + for ii in [[], [('int', i)]]: + for ff in [[], [('fields', f)]]: + args = dict(hh + bb + bble + ii + ff) + if len(args) != 0: + badtype(lambda: self.uuid.UUID(h, **args)) + if len(args) != 1: + badtype(lambda: self.uuid.UUID(**args)) + + # Immutability. + u = self.uuid.UUID(h) + badtype(lambda: setattr(u, 'hex', h)) + badtype(lambda: setattr(u, 'bytes', b)) + badtype(lambda: setattr(u, 'bytes_le', b)) + badtype(lambda: setattr(u, 'fields', f)) + badtype(lambda: setattr(u, 'int', i)) + badtype(lambda: setattr(u, 'time_low', 0)) + badtype(lambda: setattr(u, 'time_mid', 0)) + badtype(lambda: setattr(u, 'time_hi_version', 0)) + badtype(lambda: setattr(u, 'time_hi_version', 0)) + badtype(lambda: setattr(u, 'clock_seq_hi_variant', 0)) + badtype(lambda: setattr(u, 'clock_seq_low', 0)) + badtype(lambda: setattr(u, 'node', 0)) + + # Comparison with a non-UUID object + badtype(lambda: u < object()) + badtype(lambda: u > object()) + + def test_getnode(self): + node1 = self.uuid.getnode() + self.assertTrue(0 < node1 < (1 << 48), '%012x' % node1) + + # Test it again to ensure consistency. + node2 = self.uuid.getnode() + self.assertEqual(node1, node2, '%012x != %012x' % (node1, node2)) + + def test_pickle_roundtrip(self): + def check(actual, expected): + self.assertEqual(actual, expected) + self.assertEqual(actual.is_safe, expected.is_safe) + + with support.swap_item(sys.modules, 'uuid', self.uuid): + for is_safe in self.uuid.SafeUUID: + u = self.uuid.UUID('d82579ce6642a0de7ddf490a7aec7aa5', + is_safe=is_safe) + check(copy.copy(u), u) + check(copy.deepcopy(u), u) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + check(pickle.loads(pickle.dumps(u, proto)), u) + + def test_unpickle_previous_python_versions(self): + def check(actual, expected): + self.assertEqual(actual, expected) + self.assertEqual(actual.is_safe, expected.is_safe) + + pickled_uuids = [ + # Python 2.7, protocol 0 + b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' + b'tR(dS\'int\'\nL287307832597519156748809049798316161701L\nsb.', + # Python 2.7, protocol 1 + b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' + b'tR}U\x03intL287307832597519156748809049798316161701L\nsb.', + # Python 2.7, protocol 2 + b'\x80\x02cuuid\nUUID\n)\x81}U\x03int\x8a\x11\xa5z\xecz\nI\xdf}' + b'\xde\xa0Bf\xcey%\xd8\x00sb.', + # Python 3.6, protocol 0 + b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' + b'tR(dVint\nL287307832597519156748809049798316161701L\nsb.', + # Python 3.6, protocol 1 + b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' + b'tR}X\x03\x00\x00\x00intL287307832597519156748809049798316161701L' + b'\nsb.', + # Python 3.6, protocol 2 + b'\x80\x02cuuid\nUUID\n)\x81}X\x03\x00\x00\x00int\x8a\x11\xa5z\xec' + b'z\nI\xdf}\xde\xa0Bf\xcey%\xd8\x00sb.', + # Python 3.6, protocol 3 + b'\x80\x03cuuid\nUUID\n)\x81}X\x03\x00\x00\x00int\x8a\x11\xa5z\xec' + b'z\nI\xdf}\xde\xa0Bf\xcey%\xd8\x00sb.', + # Python 3.6, protocol 4 + b'\x80\x04\x95+\x00\x00\x00\x00\x00\x00\x00\x8c\x04uuid\x8c\x04UUI' + b'D\x93)\x81}\x8c\x03int\x8a\x11\xa5z\xecz\nI\xdf}\xde\xa0Bf\xcey%' + b'\xd8\x00sb.', + # Python 3.7, protocol 0 + b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' + b'tR(dVint\nL287307832597519156748809049798316161701L\nsVis_safe\n' + b'cuuid\nSafeUUID\n(NtRsb.', + # Python 3.7, protocol 1 + b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' + b'tR}(X\x03\x00\x00\x00intL287307832597519156748809049798316161701' + b'L\nX\x07\x00\x00\x00is_safecuuid\nSafeUUID\n(NtRub.', + # Python 3.7, protocol 2 + b'\x80\x02cuuid\nUUID\n)\x81}(X\x03\x00\x00\x00int\x8a\x11\xa5z' + b'\xecz\nI\xdf}\xde\xa0Bf\xcey%\xd8\x00X\x07\x00\x00\x00is_safecuu' + b'id\nSafeUUID\nN\x85Rub.', + # Python 3.7, protocol 3 + b'\x80\x03cuuid\nUUID\n)\x81}(X\x03\x00\x00\x00int\x8a\x11\xa5z' + b'\xecz\nI\xdf}\xde\xa0Bf\xcey%\xd8\x00X\x07\x00\x00\x00is_safecuu' + b'id\nSafeUUID\nN\x85Rub.', + # Python 3.7, protocol 4 + b'\x80\x04\x95F\x00\x00\x00\x00\x00\x00\x00\x8c\x04uuid\x94\x8c' + b'\x04UUID\x93)\x81}(\x8c\x03int\x8a\x11\xa5z\xecz\nI\xdf}\xde\xa0' + b'Bf\xcey%\xd8\x00\x8c\x07is_safeh\x00\x8c\x08SafeUUID\x93N\x85Rub' + b'.', + ] + pickled_uuids_safe = [ + # Python 3.7, protocol 0 + b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' + b'tR(dVint\nL287307832597519156748809049798316161701L\nsVis_safe\n' + b'cuuid\nSafeUUID\n(I0\ntRsb.', + # Python 3.7, protocol 1 + b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' + b'tR}(X\x03\x00\x00\x00intL287307832597519156748809049798316161701' + b'L\nX\x07\x00\x00\x00is_safecuuid\nSafeUUID\n(K\x00tRub.', + # Python 3.7, protocol 2 + b'\x80\x02cuuid\nUUID\n)\x81}(X\x03\x00\x00\x00int\x8a\x11\xa5z' + b'\xecz\nI\xdf}\xde\xa0Bf\xcey%\xd8\x00X\x07\x00\x00\x00is_safecuu' + b'id\nSafeUUID\nK\x00\x85Rub.', + # Python 3.7, protocol 3 + b'\x80\x03cuuid\nUUID\n)\x81}(X\x03\x00\x00\x00int\x8a\x11\xa5z' + b'\xecz\nI\xdf}\xde\xa0Bf\xcey%\xd8\x00X\x07\x00\x00\x00is_safecuu' + b'id\nSafeUUID\nK\x00\x85Rub.', + # Python 3.7, protocol 4 + b'\x80\x04\x95G\x00\x00\x00\x00\x00\x00\x00\x8c\x04uuid\x94\x8c' + b'\x04UUID\x93)\x81}(\x8c\x03int\x8a\x11\xa5z\xecz\nI\xdf}\xde\xa0' + b'Bf\xcey%\xd8\x00\x8c\x07is_safeh\x00\x8c\x08SafeUUID\x93K\x00' + b'\x85Rub.', + ] + pickled_uuids_unsafe = [ + # Python 3.7, protocol 0 + b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' + b'tR(dVint\nL287307832597519156748809049798316161701L\nsVis_safe\n' + b'cuuid\nSafeUUID\n(I-1\ntRsb.', + # Python 3.7, protocol 1 + b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' + b'tR}(X\x03\x00\x00\x00intL287307832597519156748809049798316161701' + b'L\nX\x07\x00\x00\x00is_safecuuid\nSafeUUID\n(J\xff\xff\xff\xfftR' + b'ub.', + # Python 3.7, protocol 2 + b'\x80\x02cuuid\nUUID\n)\x81}(X\x03\x00\x00\x00int\x8a\x11\xa5z' + b'\xecz\nI\xdf}\xde\xa0Bf\xcey%\xd8\x00X\x07\x00\x00\x00is_safecuu' + b'id\nSafeUUID\nJ\xff\xff\xff\xff\x85Rub.', + # Python 3.7, protocol 3 + b'\x80\x03cuuid\nUUID\n)\x81}(X\x03\x00\x00\x00int\x8a\x11\xa5z' + b'\xecz\nI\xdf}\xde\xa0Bf\xcey%\xd8\x00X\x07\x00\x00\x00is_safecuu' + b'id\nSafeUUID\nJ\xff\xff\xff\xff\x85Rub.', + # Python 3.7, protocol 4 + b'\x80\x04\x95J\x00\x00\x00\x00\x00\x00\x00\x8c\x04uuid\x94\x8c' + b'\x04UUID\x93)\x81}(\x8c\x03int\x8a\x11\xa5z\xecz\nI\xdf}\xde\xa0' + b'Bf\xcey%\xd8\x00\x8c\x07is_safeh\x00\x8c\x08SafeUUID\x93J\xff' + b'\xff\xff\xff\x85Rub.', + ] + + u = self.uuid.UUID('d82579ce6642a0de7ddf490a7aec7aa5') + u_safe = self.uuid.UUID('d82579ce6642a0de7ddf490a7aec7aa5', + is_safe=self.uuid.SafeUUID.safe) + u_unsafe = self.uuid.UUID('d82579ce6642a0de7ddf490a7aec7aa5', + is_safe=self.uuid.SafeUUID.unsafe) + + with support.swap_item(sys.modules, 'uuid', self.uuid): + for pickled in pickled_uuids: + # is_safe was added in 3.7. When unpickling values from older + # versions, is_safe will be missing, so it should be set to + # SafeUUID.unknown. + check(pickle.loads(pickled), u) + for pickled in pickled_uuids_safe: + check(pickle.loads(pickled), u_safe) + for pickled in pickled_uuids_unsafe: + check(pickle.loads(pickled), u_unsafe) + + # bpo-32502: UUID1 requires a 48-bit identifier, but hardware identifiers + # need not necessarily be 48 bits (e.g., EUI-64). + def test_uuid1_eui64(self): + # Confirm that uuid.getnode ignores hardware addresses larger than 48 + # bits. Mock out each platform's *_getnode helper functions to return + # something just larger than 48 bits to test. This will cause + # uuid.getnode to fall back on uuid._random_getnode, which will + # generate a valid value. + too_large_getter = lambda: 1 << 48 + with mock.patch.multiple( + self.uuid, + _node=None, # Ignore any cached node value. + _GETTERS=[too_large_getter], + ): + node = self.uuid.getnode() + self.assertTrue(0 < node < (1 << 48), '%012x' % node) + + # Confirm that uuid1 can use the generated node, i.e., the that + # uuid.getnode fell back on uuid._random_getnode() rather than using + # the value from too_large_getter above. + try: + self.uuid.uuid1(node=node) + except ValueError: + self.fail('uuid1 was given an invalid node ID') + + def test_uuid1(self): + equal = self.assertEqual + + # Make sure uuid1() generates UUIDs that are actually version 1. + for u in [self.uuid.uuid1() for i in range(10)]: + equal(u.variant, self.uuid.RFC_4122) + equal(u.version, 1) + self.assertIn(u.is_safe, {self.uuid.SafeUUID.safe, + self.uuid.SafeUUID.unsafe, + self.uuid.SafeUUID.unknown}) + + # Make sure the generated UUIDs are actually unique. + uuids = {} + for u in [self.uuid.uuid1() for i in range(1000)]: + uuids[u] = 1 + equal(len(uuids.keys()), 1000) + + # Make sure the supplied node ID appears in the UUID. + u = self.uuid.uuid1(0) + equal(u.node, 0) + u = self.uuid.uuid1(0x123456789abc) + equal(u.node, 0x123456789abc) + u = self.uuid.uuid1(0xffffffffffff) + equal(u.node, 0xffffffffffff) + + # Make sure the supplied clock sequence appears in the UUID. + u = self.uuid.uuid1(0x123456789abc, 0) + equal(u.node, 0x123456789abc) + equal(((u.clock_seq_hi_variant & 0x3f) << 8) | u.clock_seq_low, 0) + u = self.uuid.uuid1(0x123456789abc, 0x1234) + equal(u.node, 0x123456789abc) + equal(((u.clock_seq_hi_variant & 0x3f) << 8) | + u.clock_seq_low, 0x1234) + u = self.uuid.uuid1(0x123456789abc, 0x3fff) + equal(u.node, 0x123456789abc) + equal(((u.clock_seq_hi_variant & 0x3f) << 8) | + u.clock_seq_low, 0x3fff) + + # bpo-29925: On Mac OS X Tiger, self.uuid.uuid1().is_safe returns + # self.uuid.SafeUUID.unknown + @support.requires_mac_ver(10, 5) + @unittest.skipUnless(os.name == 'posix', 'POSIX-only test') + def test_uuid1_safe(self): + if not self.uuid._has_uuid_generate_time_safe: + self.skipTest('requires uuid_generate_time_safe(3)') + + u = self.uuid.uuid1() + # uuid_generate_time_safe() may return 0 or -1 but what it returns is + # dependent on the underlying platform support. At least it cannot be + # unknown (unless I suppose the platform is buggy). + self.assertNotEqual(u.is_safe, self.uuid.SafeUUID.unknown) + + @contextlib.contextmanager + def mock_generate_time_safe(self, safe_value): + """ + Mock uuid._generate_time_safe() to return a given *safe_value*. + """ + if os.name != 'posix': + self.skipTest('POSIX-only test') + self.uuid._load_system_functions() + f = self.uuid._generate_time_safe + if f is None: + self.skipTest('need uuid._generate_time_safe') + with mock.patch.object(self.uuid, '_generate_time_safe', + lambda: (f()[0], safe_value)): + yield + + @unittest.skipUnless(os.name == 'posix', 'POSIX-only test') + def test_uuid1_unknown(self): + # Even if the platform has uuid_generate_time_safe(), let's mock it to + # be uuid_generate_time() and ensure the safety is unknown. + with self.mock_generate_time_safe(None): + u = self.uuid.uuid1() + self.assertEqual(u.is_safe, self.uuid.SafeUUID.unknown) + + @unittest.skipUnless(os.name == 'posix', 'POSIX-only test') + def test_uuid1_is_safe(self): + with self.mock_generate_time_safe(0): + u = self.uuid.uuid1() + self.assertEqual(u.is_safe, self.uuid.SafeUUID.safe) + + @unittest.skipUnless(os.name == 'posix', 'POSIX-only test') + def test_uuid1_is_unsafe(self): + with self.mock_generate_time_safe(-1): + u = self.uuid.uuid1() + self.assertEqual(u.is_safe, self.uuid.SafeUUID.unsafe) + + @unittest.skipUnless(os.name == 'posix', 'POSIX-only test') + def test_uuid1_bogus_return_value(self): + with self.mock_generate_time_safe(3): + u = self.uuid.uuid1() + self.assertEqual(u.is_safe, self.uuid.SafeUUID.unknown) + + def test_uuid1_time(self): + with mock.patch.object(self.uuid, '_has_uuid_generate_time_safe', False), \ + mock.patch.object(self.uuid, '_generate_time_safe', None), \ + mock.patch.object(self.uuid, '_last_timestamp', None), \ + mock.patch.object(self.uuid, 'getnode', return_value=93328246233727), \ + mock.patch('time.time_ns', return_value=1545052026752910643), \ + mock.patch('random.getrandbits', return_value=5317): # guaranteed to be random + u = self.uuid.uuid1() + self.assertEqual(u, self.uuid.UUID('a7a55b92-01fc-11e9-94c5-54e1acf6da7f')) + + with mock.patch.object(self.uuid, '_has_uuid_generate_time_safe', False), \ + mock.patch.object(self.uuid, '_generate_time_safe', None), \ + mock.patch.object(self.uuid, '_last_timestamp', None), \ + mock.patch('time.time_ns', return_value=1545052026752910643): + u = self.uuid.uuid1(node=93328246233727, clock_seq=5317) + self.assertEqual(u, self.uuid.UUID('a7a55b92-01fc-11e9-94c5-54e1acf6da7f')) + + def test_uuid3(self): + equal = self.assertEqual + + # Test some known version-3 UUIDs. + for u, v in [(self.uuid.uuid3(self.uuid.NAMESPACE_DNS, 'python.org'), + '6fa459ea-ee8a-3ca4-894e-db77e160355e'), + (self.uuid.uuid3(self.uuid.NAMESPACE_URL, 'http://python.org/'), + '9fe8e8c4-aaa8-32a9-a55c-4535a88b748d'), + (self.uuid.uuid3(self.uuid.NAMESPACE_OID, '1.3.6.1'), + 'dd1a1cef-13d5-368a-ad82-eca71acd4cd1'), + (self.uuid.uuid3(self.uuid.NAMESPACE_X500, 'c=ca'), + '658d3002-db6b-3040-a1d1-8ddd7d189a4d'), + ]: + equal(u.variant, self.uuid.RFC_4122) + equal(u.version, 3) + equal(u, self.uuid.UUID(v)) + equal(str(u), v) + + def test_uuid4(self): + equal = self.assertEqual + + # Make sure uuid4() generates UUIDs that are actually version 4. + for u in [self.uuid.uuid4() for i in range(10)]: + equal(u.variant, self.uuid.RFC_4122) + equal(u.version, 4) + + # Make sure the generated UUIDs are actually unique. + uuids = {} + for u in [self.uuid.uuid4() for i in range(1000)]: + uuids[u] = 1 + equal(len(uuids.keys()), 1000) + + def test_uuid5(self): + equal = self.assertEqual + + # Test some known version-5 UUIDs. + for u, v in [(self.uuid.uuid5(self.uuid.NAMESPACE_DNS, 'python.org'), + '886313e1-3b8a-5372-9b90-0c9aee199e5d'), + (self.uuid.uuid5(self.uuid.NAMESPACE_URL, 'http://python.org/'), + '4c565f0d-3f5a-5890-b41b-20cf47701c5e'), + (self.uuid.uuid5(self.uuid.NAMESPACE_OID, '1.3.6.1'), + '1447fa61-5277-5fef-a9b3-fbc6e44f4af3'), + (self.uuid.uuid5(self.uuid.NAMESPACE_X500, 'c=ca'), + 'cc957dd1-a972-5349-98cd-874190002798'), + ]: + equal(u.variant, self.uuid.RFC_4122) + equal(u.version, 5) + equal(u, self.uuid.UUID(v)) + equal(str(u), v) + + @unittest.skipUnless(hasattr(os, 'fork'), 'need os.fork') + def testIssue8621(self): + # On at least some versions of OSX self.uuid.uuid4 generates + # the same sequence of UUIDs in the parent and any + # children started using fork. + fds = os.pipe() + pid = os.fork() + if pid == 0: + os.close(fds[0]) + value = self.uuid.uuid4() + os.write(fds[1], value.hex.encode('latin-1')) + os._exit(0) + + else: + os.close(fds[1]) + self.addCleanup(os.close, fds[0]) + parent_value = self.uuid.uuid4().hex + support.wait_process(pid, exitcode=0) + child_value = os.read(fds[0], 100).decode('latin-1') + + self.assertNotEqual(parent_value, child_value) + + def test_uuid_weakref(self): + # bpo-35701: check that weak referencing to a UUID object can be created + strong = self.uuid.uuid4() + weak = weakref.ref(strong) + self.assertIs(strong, weak()) + +class TestUUIDWithoutExtModule(BaseTestUUID, unittest.TestCase): + uuid = py_uuid + +@unittest.skipUnless(c_uuid, 'requires the C _uuid module') +class TestUUIDWithExtModule(BaseTestUUID, unittest.TestCase): + uuid = c_uuid + + +class BaseTestInternals: + _uuid = py_uuid + + def check_parse_mac(self, aix): + if not aix: + patch = mock.patch.multiple(self.uuid, + _MAC_DELIM=b':', + _MAC_OMITS_LEADING_ZEROES=False) + else: + patch = mock.patch.multiple(self.uuid, + _MAC_DELIM=b'.', + _MAC_OMITS_LEADING_ZEROES=True) + + with patch: + # Valid MAC addresses + if not aix: + tests = ( + (b'52:54:00:9d:0e:67', 0x5254009d0e67), + (b'12:34:56:78:90:ab', 0x1234567890ab), + ) + else: + # AIX format + tests = ( + (b'fe.ad.c.1.23.4', 0xfead0c012304), + ) + for mac, expected in tests: + self.assertEqual(self.uuid._parse_mac(mac), expected) + + # Invalid MAC addresses + for mac in ( + b'', + # IPv6 addresses with same length than valid MAC address + # (17 characters) + b'fe80::5054:ff:fe9', + b'123:2:3:4:5:6:7:8', + # empty 5rd field + b'52:54:00:9d::67', + # only 5 fields instead of 6 + b'52:54:00:9d:0e' + # invalid character 'x' + b'52:54:00:9d:0e:6x' + # dash separator + b'52-54-00-9d-0e-67', + ): + if aix: + mac = mac.replace(b':', b'.') + with self.subTest(mac=mac): + self.assertIsNone(self.uuid._parse_mac(mac)) + + def test_parse_mac(self): + self.check_parse_mac(False) + + def test_parse_mac_aix(self): + self.check_parse_mac(True) + + def test_find_under_heading(self): + data = '''\ +Name Mtu Network Address Ipkts Ierrs Opkts Oerrs Coll +en0 1500 link#2 fe.ad.c.1.23.4 1714807956 0 711348489 0 0 + 01:00:5e:00:00:01 +en0 1500 192.168.129 x071 1714807956 0 711348489 0 0 + 224.0.0.1 +en0 1500 192.168.90 x071 1714807956 0 711348489 0 0 + 224.0.0.1 +''' + + # The above data is from AIX - with '.' as _MAC_DELIM and strings + # shorter than 17 bytes (no leading 0). (_MAC_OMITS_LEADING_ZEROES=True) + with mock.patch.multiple(self.uuid, + _MAC_DELIM=b'.', + _MAC_OMITS_LEADING_ZEROES=True, + _get_command_stdout=mock_get_command_stdout(data)): + mac = self.uuid._find_mac_under_heading( + command='netstat', + args='-ian', + heading=b'Address', + ) + + self.assertEqual(mac, 0xfead0c012304) + + def test_find_under_heading_ipv6(self): + # bpo-39991: IPv6 address "fe80::5054:ff:fe9" looks like a MAC address + # (same string length) but must be skipped + data = '''\ +Name Mtu Network Address Ipkts Ierrs Idrop Opkts Oerrs Coll +vtnet 1500 52:54:00:9d:0e:67 10017 0 0 8174 0 0 +vtnet - fe80::%vtnet0 fe80::5054:ff:fe9 0 - - 4 - - +vtnet - 192.168.122.0 192.168.122.45 8844 - - 8171 - - +lo0 16384 lo0 260148 0 0 260148 0 0 +lo0 - ::1/128 ::1 193 - - 193 - - + ff01::1%lo0 + ff02::2:2eb7:74fa + ff02::2:ff2e:b774 + ff02::1%lo0 + ff02::1:ff00:1%lo +lo0 - fe80::%lo0/64 fe80::1%lo0 0 - - 0 - - + ff01::1%lo0 + ff02::2:2eb7:74fa + ff02::2:ff2e:b774 + ff02::1%lo0 + ff02::1:ff00:1%lo +lo0 - 127.0.0.0/8 127.0.0.1 259955 - - 259955 - - + 224.0.0.1 +''' + + with mock.patch.multiple(self.uuid, + _MAC_DELIM=b':', + _MAC_OMITS_LEADING_ZEROES=False, + _get_command_stdout=mock_get_command_stdout(data)): + mac = self.uuid._find_mac_under_heading( + command='netstat', + args='-ian', + heading=b'Address', + ) + + self.assertEqual(mac, 0x5254009d0e67) + + def test_find_mac_near_keyword(self): + # key and value are on the same line + data = ''' +fake Link encap:UNSPEC hwaddr 00-00 +cscotun0 Link encap:UNSPEC HWaddr 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00 +eth0 Link encap:Ethernet HWaddr 12:34:56:78:90:ab +''' + + # The above data will only be parsed properly on non-AIX unixes. + with mock.patch.multiple(self.uuid, + _MAC_DELIM=b':', + _MAC_OMITS_LEADING_ZEROES=False, + _get_command_stdout=mock_get_command_stdout(data)): + mac = self.uuid._find_mac_near_keyword( + command='ifconfig', + args='', + keywords=[b'hwaddr'], + get_word_index=lambda x: x + 1, + ) + + self.assertEqual(mac, 0x1234567890ab) + + def check_node(self, node, requires=None): + if requires and node is None: + self.skipTest('requires ' + requires) + hex = '%012x' % node + if support.verbose >= 2: + print(hex, end=' ') + self.assertTrue(0 < node < (1 << 48), + "%s is not an RFC 4122 node ID" % hex) + + @unittest.skipUnless(_uuid._ifconfig_getnode in _uuid._GETTERS, + "ifconfig is not used for introspection on this platform") + def test_ifconfig_getnode(self): + node = self.uuid._ifconfig_getnode() + self.check_node(node, 'ifconfig') + + @unittest.skipUnless(_uuid._ip_getnode in _uuid._GETTERS, + "ip is not used for introspection on this platform") + def test_ip_getnode(self): + node = self.uuid._ip_getnode() + self.check_node(node, 'ip') + + @unittest.skipUnless(_uuid._arp_getnode in _uuid._GETTERS, + "arp is not used for introspection on this platform") + def test_arp_getnode(self): + node = self.uuid._arp_getnode() + self.check_node(node, 'arp') + + @unittest.skipUnless(_uuid._lanscan_getnode in _uuid._GETTERS, + "lanscan is not used for introspection on this platform") + def test_lanscan_getnode(self): + node = self.uuid._lanscan_getnode() + self.check_node(node, 'lanscan') + + @unittest.skipUnless(_uuid._netstat_getnode in _uuid._GETTERS, + "netstat is not used for introspection on this platform") + def test_netstat_getnode(self): + node = self.uuid._netstat_getnode() + self.check_node(node, 'netstat') + + def test_random_getnode(self): + node = self.uuid._random_getnode() + # The multicast bit, i.e. the least significant bit of first octet, + # must be set for randomly generated MAC addresses. See RFC 4122, + # $4.1.6. + self.assertTrue(node & (1 << 40), '%012x' % node) + self.check_node(node) + + node2 = self.uuid._random_getnode() + self.assertNotEqual(node2, node, '%012x' % node) + +class TestInternalsWithoutExtModule(BaseTestInternals, unittest.TestCase): + uuid = py_uuid + +@unittest.skipUnless(c_uuid, 'requires the C _uuid module') +class TestInternalsWithExtModule(BaseTestInternals, unittest.TestCase): + uuid = c_uuid + + @unittest.skipUnless(os.name == 'posix', 'requires Posix') + def test_unix_getnode(self): + if not importable('_uuid') and not importable('ctypes'): + self.skipTest("neither _uuid extension nor ctypes available") + try: # Issues 1481, 3581: _uuid_generate_time() might be None. + node = self.uuid._unix_getnode() + except TypeError: + self.skipTest('requires uuid_generate_time') + self.check_node(node, 'unix') + + @unittest.skipUnless(os.name == 'nt', 'requires Windows') + def test_windll_getnode(self): + node = self.uuid._windll_getnode() + self.check_node(node) + + +if __name__ == '__main__': + unittest.main() From 0bb7439df39f69dbd98f0b4ad642b69a1c0c799e Mon Sep 17 00:00:00 2001 From: stu Date: Tue, 8 Nov 2022 19:26:06 +0800 Subject: [PATCH 061/137] Tests: edit uuid tests for skulpt implementation --- test/unit3/test_uuid.py | 1082 ++++++++++++++++----------------------- 1 file changed, 441 insertions(+), 641 deletions(-) diff --git a/test/unit3/test_uuid.py b/test/unit3/test_uuid.py index f5a13d2255..90d5acfc3e 100644 --- a/test/unit3/test_uuid.py +++ b/test/unit3/test_uuid.py @@ -1,32 +1,36 @@ import unittest -from test import support -from test.support import import_helper -import builtins -import contextlib -import copy -import io -import os -import pickle -import sys -import weakref -from unittest import mock - -py_uuid = import_helper.import_fresh_module('uuid', blocked=['_uuid']) -c_uuid = import_helper.import_fresh_module('uuid', fresh=['_uuid']) - -def importable(name): - try: - __import__(name) - return True - except: - return False - - -def mock_get_command_stdout(data): - def get_command_stdout(command, args): - return io.BytesIO(data.encode()) - return get_command_stdout - +# from test import support +# from test.support import import_helper +# import builtins +# import contextlib +# import copy +# import io +# import os +# import pickle +# import sys +# import weakref +# from unittest import mock + +import uuid + +# py_uuid = import_helper.import_fresh_module('uuid', blocked=['_uuid']) +# c_uuid = import_helper.import_fresh_module('uuid', fresh=['_uuid']) +py_uuid = uuid + +# def importable(name): +# try: +# __import__(name) +# return True +# except: +# return False + + +# def mock_get_command_stdout(data): +# def get_command_stdout(command, args): +# return io.BytesIO(data.encode()) +# return get_command_stdout + +bytes_ = bytes class BaseTestUUID: uuid = None @@ -166,27 +170,28 @@ def test_UUID(self): # Construct each UUID in several different ways. for u in [self.uuid.UUID(string), self.uuid.UUID(curly), self.uuid.UUID(hex), self.uuid.UUID(bytes=bytes), self.uuid.UUID(bytes_le=bytes_le), - self.uuid.UUID(fields=fields), self.uuid.UUID(int=integer), + # self.uuid.UUID(fields=fields), + self.uuid.UUID(int=integer), self.uuid.UUID(urn)]: # Test all conversions and properties of the UUID object. equal(str(u), string) equal(int(u), integer) equal(u.bytes, bytes) equal(u.bytes_le, bytes_le) - equal(u.fields, fields) - equal(u.time_low, fields[0]) - equal(u.time_mid, fields[1]) - equal(u.time_hi_version, fields[2]) - equal(u.clock_seq_hi_variant, fields[3]) - equal(u.clock_seq_low, fields[4]) - equal(u.node, fields[5]) + # equal(u.fields, fields) + # equal(u.time_low, fields[0]) + # equal(u.time_mid, fields[1]) + # equal(u.time_hi_version, fields[2]) + # equal(u.clock_seq_hi_variant, fields[3]) + # equal(u.clock_seq_low, fields[4]) + # equal(u.node, fields[5]) equal(u.hex, hex) equal(u.int, integer) equal(u.urn, urn) - equal(u.time, time) - equal(u.clock_seq, clock_seq) - equal(u.variant, variant) - equal(u.version, version) + # equal(u.time, time) + # equal(u.clock_seq, clock_seq) + # equal(u.variant, variant) + # equal(u.version, version) equivalents.append(u) # Different construction methods should give the same UUID. @@ -195,8 +200,11 @@ def test_UUID(self): equal(u, v) # Bug 7380: "bytes" and "bytes_le" should give the same type. - equal(type(u.bytes), builtins.bytes) - equal(type(u.bytes_le), builtins.bytes) + # equal(type(u.bytes), builtins.bytes) + # equal(type(u.bytes_le), builtins.bytes) + equal(type(u.bytes), bytes_) + equal(type(u.bytes_le), bytes_) + ascending.append(u) @@ -228,37 +236,37 @@ def test_exceptions(self): badvalue(lambda: self.uuid.UUID('123456781234567812345678z2345678')) # Badly formed bytes. - badvalue(lambda: self.uuid.UUID(bytes='abc')) - badvalue(lambda: self.uuid.UUID(bytes='\0'*15)) - badvalue(lambda: self.uuid.UUID(bytes='\0'*17)) + badvalue(lambda: self.uuid.UUID(bytes=b'abc')) + badvalue(lambda: self.uuid.UUID(bytes=b'\0'*15)) + badvalue(lambda: self.uuid.UUID(bytes=b'\0'*17)) # Badly formed bytes_le. - badvalue(lambda: self.uuid.UUID(bytes_le='abc')) - badvalue(lambda: self.uuid.UUID(bytes_le='\0'*15)) - badvalue(lambda: self.uuid.UUID(bytes_le='\0'*17)) - - # Badly formed fields. - badvalue(lambda: self.uuid.UUID(fields=(1,))) - badvalue(lambda: self.uuid.UUID(fields=(1, 2, 3, 4, 5))) - badvalue(lambda: self.uuid.UUID(fields=(1, 2, 3, 4, 5, 6, 7))) - - # Field values out of range. - badvalue(lambda: self.uuid.UUID(fields=(-1, 0, 0, 0, 0, 0))) - badvalue(lambda: self.uuid.UUID(fields=(0x100000000, 0, 0, 0, 0, 0))) - badvalue(lambda: self.uuid.UUID(fields=(0, -1, 0, 0, 0, 0))) - badvalue(lambda: self.uuid.UUID(fields=(0, 0x10000, 0, 0, 0, 0))) - badvalue(lambda: self.uuid.UUID(fields=(0, 0, -1, 0, 0, 0))) - badvalue(lambda: self.uuid.UUID(fields=(0, 0, 0x10000, 0, 0, 0))) - badvalue(lambda: self.uuid.UUID(fields=(0, 0, 0, -1, 0, 0))) - badvalue(lambda: self.uuid.UUID(fields=(0, 0, 0, 0x100, 0, 0))) - badvalue(lambda: self.uuid.UUID(fields=(0, 0, 0, 0, -1, 0))) - badvalue(lambda: self.uuid.UUID(fields=(0, 0, 0, 0, 0x100, 0))) - badvalue(lambda: self.uuid.UUID(fields=(0, 0, 0, 0, 0, -1))) - badvalue(lambda: self.uuid.UUID(fields=(0, 0, 0, 0, 0, 0x1000000000000))) - - # Version number out of range. - badvalue(lambda: self.uuid.UUID('00'*16, version=0)) - badvalue(lambda: self.uuid.UUID('00'*16, version=6)) + badvalue(lambda: self.uuid.UUID(bytes_le=b'abc')) + badvalue(lambda: self.uuid.UUID(bytes_le=b'\0'*15)) + badvalue(lambda: self.uuid.UUID(bytes_le=b'\0'*17)) + + # # Badly formed fields. + # badvalue(lambda: self.uuid.UUID(fields=(1,))) + # badvalue(lambda: self.uuid.UUID(fields=(1, 2, 3, 4, 5))) + # badvalue(lambda: self.uuid.UUID(fields=(1, 2, 3, 4, 5, 6, 7))) + + # # Field values out of range. + # badvalue(lambda: self.uuid.UUID(fields=(-1, 0, 0, 0, 0, 0))) + # badvalue(lambda: self.uuid.UUID(fields=(0x100000000, 0, 0, 0, 0, 0))) + # badvalue(lambda: self.uuid.UUID(fields=(0, -1, 0, 0, 0, 0))) + # badvalue(lambda: self.uuid.UUID(fields=(0, 0x10000, 0, 0, 0, 0))) + # badvalue(lambda: self.uuid.UUID(fields=(0, 0, -1, 0, 0, 0))) + # badvalue(lambda: self.uuid.UUID(fields=(0, 0, 0x10000, 0, 0, 0))) + # badvalue(lambda: self.uuid.UUID(fields=(0, 0, 0, -1, 0, 0))) + # badvalue(lambda: self.uuid.UUID(fields=(0, 0, 0, 0x100, 0, 0))) + # badvalue(lambda: self.uuid.UUID(fields=(0, 0, 0, 0, -1, 0))) + # badvalue(lambda: self.uuid.UUID(fields=(0, 0, 0, 0, 0x100, 0))) + # badvalue(lambda: self.uuid.UUID(fields=(0, 0, 0, 0, 0, -1))) + # badvalue(lambda: self.uuid.UUID(fields=(0, 0, 0, 0, 0, 0x1000000000000))) + + # # Version number out of range. + # badvalue(lambda: self.uuid.UUID('00'*16, version=0)) + # badvalue(lambda: self.uuid.UUID('00'*16, version=6)) # Integer value out of range. badvalue(lambda: self.uuid.UUID(int=-1)) @@ -270,7 +278,7 @@ def test_exceptions(self): self.uuid.UUID(hex=h) self.uuid.UUID(bytes=b) self.uuid.UUID(bytes_le=b) - self.uuid.UUID(fields=f) + # self.uuid.UUID(fields=f) self.uuid.UUID(int=i) # Wrong number of arguments (positional). @@ -294,325 +302,328 @@ def test_exceptions(self): # Immutability. u = self.uuid.UUID(h) - badtype(lambda: setattr(u, 'hex', h)) - badtype(lambda: setattr(u, 'bytes', b)) - badtype(lambda: setattr(u, 'bytes_le', b)) - badtype(lambda: setattr(u, 'fields', f)) - badtype(lambda: setattr(u, 'int', i)) - badtype(lambda: setattr(u, 'time_low', 0)) - badtype(lambda: setattr(u, 'time_mid', 0)) - badtype(lambda: setattr(u, 'time_hi_version', 0)) - badtype(lambda: setattr(u, 'time_hi_version', 0)) - badtype(lambda: setattr(u, 'clock_seq_hi_variant', 0)) - badtype(lambda: setattr(u, 'clock_seq_low', 0)) - badtype(lambda: setattr(u, 'node', 0)) + + badattr = lambda f: self.assertRaises(AttributeError, f) + + badattr(lambda: setattr(u, 'hex', h)) + badattr(lambda: setattr(u, 'bytes', b)) + badattr(lambda: setattr(u, 'bytes_le', b)) + badattr(lambda: setattr(u, 'fields', f)) + badattr(lambda: setattr(u, 'int', i)) + badattr(lambda: setattr(u, 'time_low', 0)) + badattr(lambda: setattr(u, 'time_mid', 0)) + badattr(lambda: setattr(u, 'time_hi_version', 0)) + badattr(lambda: setattr(u, 'time_hi_version', 0)) + badattr(lambda: setattr(u, 'clock_seq_hi_variant', 0)) + badattr(lambda: setattr(u, 'clock_seq_low', 0)) + badattr(lambda: setattr(u, 'node', 0)) # Comparison with a non-UUID object badtype(lambda: u < object()) badtype(lambda: u > object()) - def test_getnode(self): - node1 = self.uuid.getnode() - self.assertTrue(0 < node1 < (1 << 48), '%012x' % node1) - - # Test it again to ensure consistency. - node2 = self.uuid.getnode() - self.assertEqual(node1, node2, '%012x != %012x' % (node1, node2)) - - def test_pickle_roundtrip(self): - def check(actual, expected): - self.assertEqual(actual, expected) - self.assertEqual(actual.is_safe, expected.is_safe) - - with support.swap_item(sys.modules, 'uuid', self.uuid): - for is_safe in self.uuid.SafeUUID: - u = self.uuid.UUID('d82579ce6642a0de7ddf490a7aec7aa5', - is_safe=is_safe) - check(copy.copy(u), u) - check(copy.deepcopy(u), u) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.subTest(protocol=proto): - check(pickle.loads(pickle.dumps(u, proto)), u) - - def test_unpickle_previous_python_versions(self): - def check(actual, expected): - self.assertEqual(actual, expected) - self.assertEqual(actual.is_safe, expected.is_safe) - - pickled_uuids = [ - # Python 2.7, protocol 0 - b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' - b'tR(dS\'int\'\nL287307832597519156748809049798316161701L\nsb.', - # Python 2.7, protocol 1 - b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' - b'tR}U\x03intL287307832597519156748809049798316161701L\nsb.', - # Python 2.7, protocol 2 - b'\x80\x02cuuid\nUUID\n)\x81}U\x03int\x8a\x11\xa5z\xecz\nI\xdf}' - b'\xde\xa0Bf\xcey%\xd8\x00sb.', - # Python 3.6, protocol 0 - b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' - b'tR(dVint\nL287307832597519156748809049798316161701L\nsb.', - # Python 3.6, protocol 1 - b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' - b'tR}X\x03\x00\x00\x00intL287307832597519156748809049798316161701L' - b'\nsb.', - # Python 3.6, protocol 2 - b'\x80\x02cuuid\nUUID\n)\x81}X\x03\x00\x00\x00int\x8a\x11\xa5z\xec' - b'z\nI\xdf}\xde\xa0Bf\xcey%\xd8\x00sb.', - # Python 3.6, protocol 3 - b'\x80\x03cuuid\nUUID\n)\x81}X\x03\x00\x00\x00int\x8a\x11\xa5z\xec' - b'z\nI\xdf}\xde\xa0Bf\xcey%\xd8\x00sb.', - # Python 3.6, protocol 4 - b'\x80\x04\x95+\x00\x00\x00\x00\x00\x00\x00\x8c\x04uuid\x8c\x04UUI' - b'D\x93)\x81}\x8c\x03int\x8a\x11\xa5z\xecz\nI\xdf}\xde\xa0Bf\xcey%' - b'\xd8\x00sb.', - # Python 3.7, protocol 0 - b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' - b'tR(dVint\nL287307832597519156748809049798316161701L\nsVis_safe\n' - b'cuuid\nSafeUUID\n(NtRsb.', - # Python 3.7, protocol 1 - b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' - b'tR}(X\x03\x00\x00\x00intL287307832597519156748809049798316161701' - b'L\nX\x07\x00\x00\x00is_safecuuid\nSafeUUID\n(NtRub.', - # Python 3.7, protocol 2 - b'\x80\x02cuuid\nUUID\n)\x81}(X\x03\x00\x00\x00int\x8a\x11\xa5z' - b'\xecz\nI\xdf}\xde\xa0Bf\xcey%\xd8\x00X\x07\x00\x00\x00is_safecuu' - b'id\nSafeUUID\nN\x85Rub.', - # Python 3.7, protocol 3 - b'\x80\x03cuuid\nUUID\n)\x81}(X\x03\x00\x00\x00int\x8a\x11\xa5z' - b'\xecz\nI\xdf}\xde\xa0Bf\xcey%\xd8\x00X\x07\x00\x00\x00is_safecuu' - b'id\nSafeUUID\nN\x85Rub.', - # Python 3.7, protocol 4 - b'\x80\x04\x95F\x00\x00\x00\x00\x00\x00\x00\x8c\x04uuid\x94\x8c' - b'\x04UUID\x93)\x81}(\x8c\x03int\x8a\x11\xa5z\xecz\nI\xdf}\xde\xa0' - b'Bf\xcey%\xd8\x00\x8c\x07is_safeh\x00\x8c\x08SafeUUID\x93N\x85Rub' - b'.', - ] - pickled_uuids_safe = [ - # Python 3.7, protocol 0 - b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' - b'tR(dVint\nL287307832597519156748809049798316161701L\nsVis_safe\n' - b'cuuid\nSafeUUID\n(I0\ntRsb.', - # Python 3.7, protocol 1 - b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' - b'tR}(X\x03\x00\x00\x00intL287307832597519156748809049798316161701' - b'L\nX\x07\x00\x00\x00is_safecuuid\nSafeUUID\n(K\x00tRub.', - # Python 3.7, protocol 2 - b'\x80\x02cuuid\nUUID\n)\x81}(X\x03\x00\x00\x00int\x8a\x11\xa5z' - b'\xecz\nI\xdf}\xde\xa0Bf\xcey%\xd8\x00X\x07\x00\x00\x00is_safecuu' - b'id\nSafeUUID\nK\x00\x85Rub.', - # Python 3.7, protocol 3 - b'\x80\x03cuuid\nUUID\n)\x81}(X\x03\x00\x00\x00int\x8a\x11\xa5z' - b'\xecz\nI\xdf}\xde\xa0Bf\xcey%\xd8\x00X\x07\x00\x00\x00is_safecuu' - b'id\nSafeUUID\nK\x00\x85Rub.', - # Python 3.7, protocol 4 - b'\x80\x04\x95G\x00\x00\x00\x00\x00\x00\x00\x8c\x04uuid\x94\x8c' - b'\x04UUID\x93)\x81}(\x8c\x03int\x8a\x11\xa5z\xecz\nI\xdf}\xde\xa0' - b'Bf\xcey%\xd8\x00\x8c\x07is_safeh\x00\x8c\x08SafeUUID\x93K\x00' - b'\x85Rub.', - ] - pickled_uuids_unsafe = [ - # Python 3.7, protocol 0 - b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' - b'tR(dVint\nL287307832597519156748809049798316161701L\nsVis_safe\n' - b'cuuid\nSafeUUID\n(I-1\ntRsb.', - # Python 3.7, protocol 1 - b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' - b'tR}(X\x03\x00\x00\x00intL287307832597519156748809049798316161701' - b'L\nX\x07\x00\x00\x00is_safecuuid\nSafeUUID\n(J\xff\xff\xff\xfftR' - b'ub.', - # Python 3.7, protocol 2 - b'\x80\x02cuuid\nUUID\n)\x81}(X\x03\x00\x00\x00int\x8a\x11\xa5z' - b'\xecz\nI\xdf}\xde\xa0Bf\xcey%\xd8\x00X\x07\x00\x00\x00is_safecuu' - b'id\nSafeUUID\nJ\xff\xff\xff\xff\x85Rub.', - # Python 3.7, protocol 3 - b'\x80\x03cuuid\nUUID\n)\x81}(X\x03\x00\x00\x00int\x8a\x11\xa5z' - b'\xecz\nI\xdf}\xde\xa0Bf\xcey%\xd8\x00X\x07\x00\x00\x00is_safecuu' - b'id\nSafeUUID\nJ\xff\xff\xff\xff\x85Rub.', - # Python 3.7, protocol 4 - b'\x80\x04\x95J\x00\x00\x00\x00\x00\x00\x00\x8c\x04uuid\x94\x8c' - b'\x04UUID\x93)\x81}(\x8c\x03int\x8a\x11\xa5z\xecz\nI\xdf}\xde\xa0' - b'Bf\xcey%\xd8\x00\x8c\x07is_safeh\x00\x8c\x08SafeUUID\x93J\xff' - b'\xff\xff\xff\x85Rub.', - ] - - u = self.uuid.UUID('d82579ce6642a0de7ddf490a7aec7aa5') - u_safe = self.uuid.UUID('d82579ce6642a0de7ddf490a7aec7aa5', - is_safe=self.uuid.SafeUUID.safe) - u_unsafe = self.uuid.UUID('d82579ce6642a0de7ddf490a7aec7aa5', - is_safe=self.uuid.SafeUUID.unsafe) - - with support.swap_item(sys.modules, 'uuid', self.uuid): - for pickled in pickled_uuids: - # is_safe was added in 3.7. When unpickling values from older - # versions, is_safe will be missing, so it should be set to - # SafeUUID.unknown. - check(pickle.loads(pickled), u) - for pickled in pickled_uuids_safe: - check(pickle.loads(pickled), u_safe) - for pickled in pickled_uuids_unsafe: - check(pickle.loads(pickled), u_unsafe) - - # bpo-32502: UUID1 requires a 48-bit identifier, but hardware identifiers - # need not necessarily be 48 bits (e.g., EUI-64). - def test_uuid1_eui64(self): - # Confirm that uuid.getnode ignores hardware addresses larger than 48 - # bits. Mock out each platform's *_getnode helper functions to return - # something just larger than 48 bits to test. This will cause - # uuid.getnode to fall back on uuid._random_getnode, which will - # generate a valid value. - too_large_getter = lambda: 1 << 48 - with mock.patch.multiple( - self.uuid, - _node=None, # Ignore any cached node value. - _GETTERS=[too_large_getter], - ): - node = self.uuid.getnode() - self.assertTrue(0 < node < (1 << 48), '%012x' % node) - - # Confirm that uuid1 can use the generated node, i.e., the that - # uuid.getnode fell back on uuid._random_getnode() rather than using - # the value from too_large_getter above. - try: - self.uuid.uuid1(node=node) - except ValueError: - self.fail('uuid1 was given an invalid node ID') - - def test_uuid1(self): - equal = self.assertEqual - - # Make sure uuid1() generates UUIDs that are actually version 1. - for u in [self.uuid.uuid1() for i in range(10)]: - equal(u.variant, self.uuid.RFC_4122) - equal(u.version, 1) - self.assertIn(u.is_safe, {self.uuid.SafeUUID.safe, - self.uuid.SafeUUID.unsafe, - self.uuid.SafeUUID.unknown}) - - # Make sure the generated UUIDs are actually unique. - uuids = {} - for u in [self.uuid.uuid1() for i in range(1000)]: - uuids[u] = 1 - equal(len(uuids.keys()), 1000) - - # Make sure the supplied node ID appears in the UUID. - u = self.uuid.uuid1(0) - equal(u.node, 0) - u = self.uuid.uuid1(0x123456789abc) - equal(u.node, 0x123456789abc) - u = self.uuid.uuid1(0xffffffffffff) - equal(u.node, 0xffffffffffff) - - # Make sure the supplied clock sequence appears in the UUID. - u = self.uuid.uuid1(0x123456789abc, 0) - equal(u.node, 0x123456789abc) - equal(((u.clock_seq_hi_variant & 0x3f) << 8) | u.clock_seq_low, 0) - u = self.uuid.uuid1(0x123456789abc, 0x1234) - equal(u.node, 0x123456789abc) - equal(((u.clock_seq_hi_variant & 0x3f) << 8) | - u.clock_seq_low, 0x1234) - u = self.uuid.uuid1(0x123456789abc, 0x3fff) - equal(u.node, 0x123456789abc) - equal(((u.clock_seq_hi_variant & 0x3f) << 8) | - u.clock_seq_low, 0x3fff) - - # bpo-29925: On Mac OS X Tiger, self.uuid.uuid1().is_safe returns - # self.uuid.SafeUUID.unknown - @support.requires_mac_ver(10, 5) - @unittest.skipUnless(os.name == 'posix', 'POSIX-only test') - def test_uuid1_safe(self): - if not self.uuid._has_uuid_generate_time_safe: - self.skipTest('requires uuid_generate_time_safe(3)') - - u = self.uuid.uuid1() - # uuid_generate_time_safe() may return 0 or -1 but what it returns is - # dependent on the underlying platform support. At least it cannot be - # unknown (unless I suppose the platform is buggy). - self.assertNotEqual(u.is_safe, self.uuid.SafeUUID.unknown) - - @contextlib.contextmanager - def mock_generate_time_safe(self, safe_value): - """ - Mock uuid._generate_time_safe() to return a given *safe_value*. - """ - if os.name != 'posix': - self.skipTest('POSIX-only test') - self.uuid._load_system_functions() - f = self.uuid._generate_time_safe - if f is None: - self.skipTest('need uuid._generate_time_safe') - with mock.patch.object(self.uuid, '_generate_time_safe', - lambda: (f()[0], safe_value)): - yield - - @unittest.skipUnless(os.name == 'posix', 'POSIX-only test') - def test_uuid1_unknown(self): - # Even if the platform has uuid_generate_time_safe(), let's mock it to - # be uuid_generate_time() and ensure the safety is unknown. - with self.mock_generate_time_safe(None): - u = self.uuid.uuid1() - self.assertEqual(u.is_safe, self.uuid.SafeUUID.unknown) - - @unittest.skipUnless(os.name == 'posix', 'POSIX-only test') - def test_uuid1_is_safe(self): - with self.mock_generate_time_safe(0): - u = self.uuid.uuid1() - self.assertEqual(u.is_safe, self.uuid.SafeUUID.safe) - - @unittest.skipUnless(os.name == 'posix', 'POSIX-only test') - def test_uuid1_is_unsafe(self): - with self.mock_generate_time_safe(-1): - u = self.uuid.uuid1() - self.assertEqual(u.is_safe, self.uuid.SafeUUID.unsafe) - - @unittest.skipUnless(os.name == 'posix', 'POSIX-only test') - def test_uuid1_bogus_return_value(self): - with self.mock_generate_time_safe(3): - u = self.uuid.uuid1() - self.assertEqual(u.is_safe, self.uuid.SafeUUID.unknown) - - def test_uuid1_time(self): - with mock.patch.object(self.uuid, '_has_uuid_generate_time_safe', False), \ - mock.patch.object(self.uuid, '_generate_time_safe', None), \ - mock.patch.object(self.uuid, '_last_timestamp', None), \ - mock.patch.object(self.uuid, 'getnode', return_value=93328246233727), \ - mock.patch('time.time_ns', return_value=1545052026752910643), \ - mock.patch('random.getrandbits', return_value=5317): # guaranteed to be random - u = self.uuid.uuid1() - self.assertEqual(u, self.uuid.UUID('a7a55b92-01fc-11e9-94c5-54e1acf6da7f')) - - with mock.patch.object(self.uuid, '_has_uuid_generate_time_safe', False), \ - mock.patch.object(self.uuid, '_generate_time_safe', None), \ - mock.patch.object(self.uuid, '_last_timestamp', None), \ - mock.patch('time.time_ns', return_value=1545052026752910643): - u = self.uuid.uuid1(node=93328246233727, clock_seq=5317) - self.assertEqual(u, self.uuid.UUID('a7a55b92-01fc-11e9-94c5-54e1acf6da7f')) - - def test_uuid3(self): - equal = self.assertEqual - - # Test some known version-3 UUIDs. - for u, v in [(self.uuid.uuid3(self.uuid.NAMESPACE_DNS, 'python.org'), - '6fa459ea-ee8a-3ca4-894e-db77e160355e'), - (self.uuid.uuid3(self.uuid.NAMESPACE_URL, 'http://python.org/'), - '9fe8e8c4-aaa8-32a9-a55c-4535a88b748d'), - (self.uuid.uuid3(self.uuid.NAMESPACE_OID, '1.3.6.1'), - 'dd1a1cef-13d5-368a-ad82-eca71acd4cd1'), - (self.uuid.uuid3(self.uuid.NAMESPACE_X500, 'c=ca'), - '658d3002-db6b-3040-a1d1-8ddd7d189a4d'), - ]: - equal(u.variant, self.uuid.RFC_4122) - equal(u.version, 3) - equal(u, self.uuid.UUID(v)) - equal(str(u), v) + # def test_getnode(self): + # node1 = self.uuid.getnode() + # self.assertTrue(0 < node1 < (1 << 48), '%012x' % node1) + + # # Test it again to ensure consistency. + # node2 = self.uuid.getnode() + # self.assertEqual(node1, node2, '%012x != %012x' % (node1, node2)) + + # def test_pickle_roundtrip(self): + # def check(actual, expected): + # self.assertEqual(actual, expected) + # self.assertEqual(actual.is_safe, expected.is_safe) + + # with support.swap_item(sys.modules, 'uuid', self.uuid): + # for is_safe in self.uuid.SafeUUID: + # u = self.uuid.UUID('d82579ce6642a0de7ddf490a7aec7aa5', + # is_safe=is_safe) + # check(copy.copy(u), u) + # check(copy.deepcopy(u), u) + # for proto in range(pickle.HIGHEST_PROTOCOL + 1): + # with self.subTest(protocol=proto): + # check(pickle.loads(pickle.dumps(u, proto)), u) + + # def test_unpickle_previous_python_versions(self): + # def check(actual, expected): + # self.assertEqual(actual, expected) + # self.assertEqual(actual.is_safe, expected.is_safe) + + # pickled_uuids = [ + # # Python 2.7, protocol 0 + # b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' + # b'tR(dS\'int\'\nL287307832597519156748809049798316161701L\nsb.', + # # Python 2.7, protocol 1 + # b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' + # b'tR}U\x03intL287307832597519156748809049798316161701L\nsb.', + # # Python 2.7, protocol 2 + # b'\x80\x02cuuid\nUUID\n)\x81}U\x03int\x8a\x11\xa5z\xecz\nI\xdf}' + # b'\xde\xa0Bf\xcey%\xd8\x00sb.', + # # Python 3.6, protocol 0 + # b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' + # b'tR(dVint\nL287307832597519156748809049798316161701L\nsb.', + # # Python 3.6, protocol 1 + # b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' + # b'tR}X\x03\x00\x00\x00intL287307832597519156748809049798316161701L' + # b'\nsb.', + # # Python 3.6, protocol 2 + # b'\x80\x02cuuid\nUUID\n)\x81}X\x03\x00\x00\x00int\x8a\x11\xa5z\xec' + # b'z\nI\xdf}\xde\xa0Bf\xcey%\xd8\x00sb.', + # # Python 3.6, protocol 3 + # b'\x80\x03cuuid\nUUID\n)\x81}X\x03\x00\x00\x00int\x8a\x11\xa5z\xec' + # b'z\nI\xdf}\xde\xa0Bf\xcey%\xd8\x00sb.', + # # Python 3.6, protocol 4 + # b'\x80\x04\x95+\x00\x00\x00\x00\x00\x00\x00\x8c\x04uuid\x8c\x04UUI' + # b'D\x93)\x81}\x8c\x03int\x8a\x11\xa5z\xecz\nI\xdf}\xde\xa0Bf\xcey%' + # b'\xd8\x00sb.', + # # Python 3.7, protocol 0 + # b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' + # b'tR(dVint\nL287307832597519156748809049798316161701L\nsVis_safe\n' + # b'cuuid\nSafeUUID\n(NtRsb.', + # # Python 3.7, protocol 1 + # b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' + # b'tR}(X\x03\x00\x00\x00intL287307832597519156748809049798316161701' + # b'L\nX\x07\x00\x00\x00is_safecuuid\nSafeUUID\n(NtRub.', + # # Python 3.7, protocol 2 + # b'\x80\x02cuuid\nUUID\n)\x81}(X\x03\x00\x00\x00int\x8a\x11\xa5z' + # b'\xecz\nI\xdf}\xde\xa0Bf\xcey%\xd8\x00X\x07\x00\x00\x00is_safecuu' + # b'id\nSafeUUID\nN\x85Rub.', + # # Python 3.7, protocol 3 + # b'\x80\x03cuuid\nUUID\n)\x81}(X\x03\x00\x00\x00int\x8a\x11\xa5z' + # b'\xecz\nI\xdf}\xde\xa0Bf\xcey%\xd8\x00X\x07\x00\x00\x00is_safecuu' + # b'id\nSafeUUID\nN\x85Rub.', + # # Python 3.7, protocol 4 + # b'\x80\x04\x95F\x00\x00\x00\x00\x00\x00\x00\x8c\x04uuid\x94\x8c' + # b'\x04UUID\x93)\x81}(\x8c\x03int\x8a\x11\xa5z\xecz\nI\xdf}\xde\xa0' + # b'Bf\xcey%\xd8\x00\x8c\x07is_safeh\x00\x8c\x08SafeUUID\x93N\x85Rub' + # b'.', + # ] + # pickled_uuids_safe = [ + # # Python 3.7, protocol 0 + # b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' + # b'tR(dVint\nL287307832597519156748809049798316161701L\nsVis_safe\n' + # b'cuuid\nSafeUUID\n(I0\ntRsb.', + # # Python 3.7, protocol 1 + # b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' + # b'tR}(X\x03\x00\x00\x00intL287307832597519156748809049798316161701' + # b'L\nX\x07\x00\x00\x00is_safecuuid\nSafeUUID\n(K\x00tRub.', + # # Python 3.7, protocol 2 + # b'\x80\x02cuuid\nUUID\n)\x81}(X\x03\x00\x00\x00int\x8a\x11\xa5z' + # b'\xecz\nI\xdf}\xde\xa0Bf\xcey%\xd8\x00X\x07\x00\x00\x00is_safecuu' + # b'id\nSafeUUID\nK\x00\x85Rub.', + # # Python 3.7, protocol 3 + # b'\x80\x03cuuid\nUUID\n)\x81}(X\x03\x00\x00\x00int\x8a\x11\xa5z' + # b'\xecz\nI\xdf}\xde\xa0Bf\xcey%\xd8\x00X\x07\x00\x00\x00is_safecuu' + # b'id\nSafeUUID\nK\x00\x85Rub.', + # # Python 3.7, protocol 4 + # b'\x80\x04\x95G\x00\x00\x00\x00\x00\x00\x00\x8c\x04uuid\x94\x8c' + # b'\x04UUID\x93)\x81}(\x8c\x03int\x8a\x11\xa5z\xecz\nI\xdf}\xde\xa0' + # b'Bf\xcey%\xd8\x00\x8c\x07is_safeh\x00\x8c\x08SafeUUID\x93K\x00' + # b'\x85Rub.', + # ] + # pickled_uuids_unsafe = [ + # # Python 3.7, protocol 0 + # b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' + # b'tR(dVint\nL287307832597519156748809049798316161701L\nsVis_safe\n' + # b'cuuid\nSafeUUID\n(I-1\ntRsb.', + # # Python 3.7, protocol 1 + # b'ccopy_reg\n_reconstructor\n(cuuid\nUUID\nc__builtin__\nobject\nN' + # b'tR}(X\x03\x00\x00\x00intL287307832597519156748809049798316161701' + # b'L\nX\x07\x00\x00\x00is_safecuuid\nSafeUUID\n(J\xff\xff\xff\xfftR' + # b'ub.', + # # Python 3.7, protocol 2 + # b'\x80\x02cuuid\nUUID\n)\x81}(X\x03\x00\x00\x00int\x8a\x11\xa5z' + # b'\xecz\nI\xdf}\xde\xa0Bf\xcey%\xd8\x00X\x07\x00\x00\x00is_safecuu' + # b'id\nSafeUUID\nJ\xff\xff\xff\xff\x85Rub.', + # # Python 3.7, protocol 3 + # b'\x80\x03cuuid\nUUID\n)\x81}(X\x03\x00\x00\x00int\x8a\x11\xa5z' + # b'\xecz\nI\xdf}\xde\xa0Bf\xcey%\xd8\x00X\x07\x00\x00\x00is_safecuu' + # b'id\nSafeUUID\nJ\xff\xff\xff\xff\x85Rub.', + # # Python 3.7, protocol 4 + # b'\x80\x04\x95J\x00\x00\x00\x00\x00\x00\x00\x8c\x04uuid\x94\x8c' + # b'\x04UUID\x93)\x81}(\x8c\x03int\x8a\x11\xa5z\xecz\nI\xdf}\xde\xa0' + # b'Bf\xcey%\xd8\x00\x8c\x07is_safeh\x00\x8c\x08SafeUUID\x93J\xff' + # b'\xff\xff\xff\x85Rub.', + # ] + + # u = self.uuid.UUID('d82579ce6642a0de7ddf490a7aec7aa5') + # u_safe = self.uuid.UUID('d82579ce6642a0de7ddf490a7aec7aa5', + # is_safe=self.uuid.SafeUUID.safe) + # u_unsafe = self.uuid.UUID('d82579ce6642a0de7ddf490a7aec7aa5', + # is_safe=self.uuid.SafeUUID.unsafe) + + # with support.swap_item(sys.modules, 'uuid', self.uuid): + # for pickled in pickled_uuids: + # # is_safe was added in 3.7. When unpickling values from older + # # versions, is_safe will be missing, so it should be set to + # # SafeUUID.unknown. + # check(pickle.loads(pickled), u) + # for pickled in pickled_uuids_safe: + # check(pickle.loads(pickled), u_safe) + # for pickled in pickled_uuids_unsafe: + # check(pickle.loads(pickled), u_unsafe) + + # # bpo-32502: UUID1 requires a 48-bit identifier, but hardware identifiers + # # need not necessarily be 48 bits (e.g., EUI-64). + # def test_uuid1_eui64(self): + # # Confirm that uuid.getnode ignores hardware addresses larger than 48 + # # bits. Mock out each platform's *_getnode helper functions to return + # # something just larger than 48 bits to test. This will cause + # # uuid.getnode to fall back on uuid._random_getnode, which will + # # generate a valid value. + # too_large_getter = lambda: 1 << 48 + # with mock.patch.multiple( + # self.uuid, + # _node=None, # Ignore any cached node value. + # _GETTERS=[too_large_getter], + # ): + # node = self.uuid.getnode() + # self.assertTrue(0 < node < (1 << 48), '%012x' % node) + + # # Confirm that uuid1 can use the generated node, i.e., the that + # # uuid.getnode fell back on uuid._random_getnode() rather than using + # # the value from too_large_getter above. + # try: + # self.uuid.uuid1(node=node) + # except ValueError: + # self.fail('uuid1 was given an invalid node ID') + + # def test_uuid1(self): + # equal = self.assertEqual + + # # Make sure uuid1() generates UUIDs that are actually version 1. + # for u in [self.uuid.uuid1() for i in range(10)]: + # equal(u.variant, self.uuid.RFC_4122) + # equal(u.version, 1) + # self.assertIn(u.is_safe, {self.uuid.SafeUUID.safe, + # self.uuid.SafeUUID.unsafe, + # self.uuid.SafeUUID.unknown}) + + # # Make sure the generated UUIDs are actually unique. + # uuids = {} + # for u in [self.uuid.uuid1() for i in range(1000)]: + # uuids[u] = 1 + # equal(len(uuids.keys()), 1000) + + # # Make sure the supplied node ID appears in the UUID. + # u = self.uuid.uuid1(0) + # equal(u.node, 0) + # u = self.uuid.uuid1(0x123456789abc) + # equal(u.node, 0x123456789abc) + # u = self.uuid.uuid1(0xffffffffffff) + # equal(u.node, 0xffffffffffff) + + # # Make sure the supplied clock sequence appears in the UUID. + # u = self.uuid.uuid1(0x123456789abc, 0) + # equal(u.node, 0x123456789abc) + # equal(((u.clock_seq_hi_variant & 0x3f) << 8) | u.clock_seq_low, 0) + # u = self.uuid.uuid1(0x123456789abc, 0x1234) + # equal(u.node, 0x123456789abc) + # equal(((u.clock_seq_hi_variant & 0x3f) << 8) | + # u.clock_seq_low, 0x1234) + # u = self.uuid.uuid1(0x123456789abc, 0x3fff) + # equal(u.node, 0x123456789abc) + # equal(((u.clock_seq_hi_variant & 0x3f) << 8) | + # u.clock_seq_low, 0x3fff) + + # # bpo-29925: On Mac OS X Tiger, self.uuid.uuid1().is_safe returns + # # self.uuid.SafeUUID.unknown + # @support.requires_mac_ver(10, 5) + # @unittest.skipUnless(os.name == 'posix', 'POSIX-only test') + # def test_uuid1_safe(self): + # if not self.uuid._has_uuid_generate_time_safe: + # self.skipTest('requires uuid_generate_time_safe(3)') + + # u = self.uuid.uuid1() + # # uuid_generate_time_safe() may return 0 or -1 but what it returns is + # # dependent on the underlying platform support. At least it cannot be + # # unknown (unless I suppose the platform is buggy). + # self.assertNotEqual(u.is_safe, self.uuid.SafeUUID.unknown) + + # @contextlib.contextmanager + # def mock_generate_time_safe(self, safe_value): + # """ + # Mock uuid._generate_time_safe() to return a given *safe_value*. + # """ + # if os.name != 'posix': + # self.skipTest('POSIX-only test') + # self.uuid._load_system_functions() + # f = self.uuid._generate_time_safe + # if f is None: + # self.skipTest('need uuid._generate_time_safe') + # with mock.patch.object(self.uuid, '_generate_time_safe', + # lambda: (f()[0], safe_value)): + # yield + + # @unittest.skipUnless(os.name == 'posix', 'POSIX-only test') + # def test_uuid1_unknown(self): + # # Even if the platform has uuid_generate_time_safe(), let's mock it to + # # be uuid_generate_time() and ensure the safety is unknown. + # with self.mock_generate_time_safe(None): + # u = self.uuid.uuid1() + # self.assertEqual(u.is_safe, self.uuid.SafeUUID.unknown) + + # @unittest.skipUnless(os.name == 'posix', 'POSIX-only test') + # def test_uuid1_is_safe(self): + # with self.mock_generate_time_safe(0): + # u = self.uuid.uuid1() + # self.assertEqual(u.is_safe, self.uuid.SafeUUID.safe) + + # @unittest.skipUnless(os.name == 'posix', 'POSIX-only test') + # def test_uuid1_is_unsafe(self): + # with self.mock_generate_time_safe(-1): + # u = self.uuid.uuid1() + # self.assertEqual(u.is_safe, self.uuid.SafeUUID.unsafe) + + # @unittest.skipUnless(os.name == 'posix', 'POSIX-only test') + # def test_uuid1_bogus_return_value(self): + # with self.mock_generate_time_safe(3): + # u = self.uuid.uuid1() + # self.assertEqual(u.is_safe, self.uuid.SafeUUID.unknown) + + # def test_uuid1_time(self): + # with mock.patch.object(self.uuid, '_has_uuid_generate_time_safe', False), \ + # mock.patch.object(self.uuid, '_generate_time_safe', None), \ + # mock.patch.object(self.uuid, '_last_timestamp', None), \ + # mock.patch.object(self.uuid, 'getnode', return_value=93328246233727), \ + # mock.patch('time.time_ns', return_value=1545052026752910643), \ + # mock.patch('random.getrandbits', return_value=5317): # guaranteed to be random + # u = self.uuid.uuid1() + # self.assertEqual(u, self.uuid.UUID('a7a55b92-01fc-11e9-94c5-54e1acf6da7f')) + + # with mock.patch.object(self.uuid, '_has_uuid_generate_time_safe', False), \ + # mock.patch.object(self.uuid, '_generate_time_safe', None), \ + # mock.patch.object(self.uuid, '_last_timestamp', None), \ + # mock.patch('time.time_ns', return_value=1545052026752910643): + # u = self.uuid.uuid1(node=93328246233727, clock_seq=5317) + # self.assertEqual(u, self.uuid.UUID('a7a55b92-01fc-11e9-94c5-54e1acf6da7f')) + + # def test_uuid3(self): + # equal = self.assertEqual + + # # Test some known version-3 UUIDs. + # for u, v in [(self.uuid.uuid3(self.uuid.NAMESPACE_DNS, 'python.org'), + # '6fa459ea-ee8a-3ca4-894e-db77e160355e'), + # (self.uuid.uuid3(self.uuid.NAMESPACE_URL, 'http://python.org/'), + # '9fe8e8c4-aaa8-32a9-a55c-4535a88b748d'), + # (self.uuid.uuid3(self.uuid.NAMESPACE_OID, '1.3.6.1'), + # 'dd1a1cef-13d5-368a-ad82-eca71acd4cd1'), + # (self.uuid.uuid3(self.uuid.NAMESPACE_X500, 'c=ca'), + # '658d3002-db6b-3040-a1d1-8ddd7d189a4d'), + # ]: + # equal(u.variant, self.uuid.RFC_4122) + # equal(u.version, 3) + # equal(u, self.uuid.UUID(v)) + # equal(str(u), v) def test_uuid4(self): equal = self.assertEqual # Make sure uuid4() generates UUIDs that are actually version 4. - for u in [self.uuid.uuid4() for i in range(10)]: - equal(u.variant, self.uuid.RFC_4122) - equal(u.version, 4) + # for u in [self.uuid.uuid4() for i in range(10)]: + # equal(u.variant, self.uuid.RFC_4122) + # equal(u.version, 4) # Make sure the generated UUIDs are actually unique. uuids = {} @@ -620,271 +631,60 @@ def test_uuid4(self): uuids[u] = 1 equal(len(uuids.keys()), 1000) - def test_uuid5(self): - equal = self.assertEqual - - # Test some known version-5 UUIDs. - for u, v in [(self.uuid.uuid5(self.uuid.NAMESPACE_DNS, 'python.org'), - '886313e1-3b8a-5372-9b90-0c9aee199e5d'), - (self.uuid.uuid5(self.uuid.NAMESPACE_URL, 'http://python.org/'), - '4c565f0d-3f5a-5890-b41b-20cf47701c5e'), - (self.uuid.uuid5(self.uuid.NAMESPACE_OID, '1.3.6.1'), - '1447fa61-5277-5fef-a9b3-fbc6e44f4af3'), - (self.uuid.uuid5(self.uuid.NAMESPACE_X500, 'c=ca'), - 'cc957dd1-a972-5349-98cd-874190002798'), - ]: - equal(u.variant, self.uuid.RFC_4122) - equal(u.version, 5) - equal(u, self.uuid.UUID(v)) - equal(str(u), v) - - @unittest.skipUnless(hasattr(os, 'fork'), 'need os.fork') - def testIssue8621(self): - # On at least some versions of OSX self.uuid.uuid4 generates - # the same sequence of UUIDs in the parent and any - # children started using fork. - fds = os.pipe() - pid = os.fork() - if pid == 0: - os.close(fds[0]) - value = self.uuid.uuid4() - os.write(fds[1], value.hex.encode('latin-1')) - os._exit(0) - - else: - os.close(fds[1]) - self.addCleanup(os.close, fds[0]) - parent_value = self.uuid.uuid4().hex - support.wait_process(pid, exitcode=0) - child_value = os.read(fds[0], 100).decode('latin-1') - - self.assertNotEqual(parent_value, child_value) - - def test_uuid_weakref(self): - # bpo-35701: check that weak referencing to a UUID object can be created - strong = self.uuid.uuid4() - weak = weakref.ref(strong) - self.assertIs(strong, weak()) + # def test_uuid5(self): + # equal = self.assertEqual + + # # Test some known version-5 UUIDs. + # for u, v in [(self.uuid.uuid5(self.uuid.NAMESPACE_DNS, 'python.org'), + # '886313e1-3b8a-5372-9b90-0c9aee199e5d'), + # (self.uuid.uuid5(self.uuid.NAMESPACE_URL, 'http://python.org/'), + # '4c565f0d-3f5a-5890-b41b-20cf47701c5e'), + # (self.uuid.uuid5(self.uuid.NAMESPACE_OID, '1.3.6.1'), + # '1447fa61-5277-5fef-a9b3-fbc6e44f4af3'), + # (self.uuid.uuid5(self.uuid.NAMESPACE_X500, 'c=ca'), + # 'cc957dd1-a972-5349-98cd-874190002798'), + # ]: + # equal(u.variant, self.uuid.RFC_4122) + # equal(u.version, 5) + # equal(u, self.uuid.UUID(v)) + # equal(str(u), v) + + # @unittest.skipUnless(hasattr(os, 'fork'), 'need os.fork') + # def testIssue8621(self): + # # On at least some versions of OSX self.uuid.uuid4 generates + # # the same sequence of UUIDs in the parent and any + # # children started using fork. + # fds = os.pipe() + # pid = os.fork() + # if pid == 0: + # os.close(fds[0]) + # value = self.uuid.uuid4() + # os.write(fds[1], value.hex.encode('latin-1')) + # os._exit(0) + + # else: + # os.close(fds[1]) + # self.addCleanup(os.close, fds[0]) + # parent_value = self.uuid.uuid4().hex + # support.wait_process(pid, exitcode=0) + # child_value = os.read(fds[0], 100).decode('latin-1') + + # self.assertNotEqual(parent_value, child_value) + + # def test_uuid_weakref(self): + # # bpo-35701: check that weak referencing to a UUID object can be created + # strong = self.uuid.uuid4() + # weak = weakref.ref(strong) + # self.assertIs(strong, weak()) class TestUUIDWithoutExtModule(BaseTestUUID, unittest.TestCase): uuid = py_uuid -@unittest.skipUnless(c_uuid, 'requires the C _uuid module') -class TestUUIDWithExtModule(BaseTestUUID, unittest.TestCase): - uuid = c_uuid - - -class BaseTestInternals: - _uuid = py_uuid - - def check_parse_mac(self, aix): - if not aix: - patch = mock.patch.multiple(self.uuid, - _MAC_DELIM=b':', - _MAC_OMITS_LEADING_ZEROES=False) - else: - patch = mock.patch.multiple(self.uuid, - _MAC_DELIM=b'.', - _MAC_OMITS_LEADING_ZEROES=True) - - with patch: - # Valid MAC addresses - if not aix: - tests = ( - (b'52:54:00:9d:0e:67', 0x5254009d0e67), - (b'12:34:56:78:90:ab', 0x1234567890ab), - ) - else: - # AIX format - tests = ( - (b'fe.ad.c.1.23.4', 0xfead0c012304), - ) - for mac, expected in tests: - self.assertEqual(self.uuid._parse_mac(mac), expected) - - # Invalid MAC addresses - for mac in ( - b'', - # IPv6 addresses with same length than valid MAC address - # (17 characters) - b'fe80::5054:ff:fe9', - b'123:2:3:4:5:6:7:8', - # empty 5rd field - b'52:54:00:9d::67', - # only 5 fields instead of 6 - b'52:54:00:9d:0e' - # invalid character 'x' - b'52:54:00:9d:0e:6x' - # dash separator - b'52-54-00-9d-0e-67', - ): - if aix: - mac = mac.replace(b':', b'.') - with self.subTest(mac=mac): - self.assertIsNone(self.uuid._parse_mac(mac)) - - def test_parse_mac(self): - self.check_parse_mac(False) - - def test_parse_mac_aix(self): - self.check_parse_mac(True) - - def test_find_under_heading(self): - data = '''\ -Name Mtu Network Address Ipkts Ierrs Opkts Oerrs Coll -en0 1500 link#2 fe.ad.c.1.23.4 1714807956 0 711348489 0 0 - 01:00:5e:00:00:01 -en0 1500 192.168.129 x071 1714807956 0 711348489 0 0 - 224.0.0.1 -en0 1500 192.168.90 x071 1714807956 0 711348489 0 0 - 224.0.0.1 -''' - - # The above data is from AIX - with '.' as _MAC_DELIM and strings - # shorter than 17 bytes (no leading 0). (_MAC_OMITS_LEADING_ZEROES=True) - with mock.patch.multiple(self.uuid, - _MAC_DELIM=b'.', - _MAC_OMITS_LEADING_ZEROES=True, - _get_command_stdout=mock_get_command_stdout(data)): - mac = self.uuid._find_mac_under_heading( - command='netstat', - args='-ian', - heading=b'Address', - ) - - self.assertEqual(mac, 0xfead0c012304) - - def test_find_under_heading_ipv6(self): - # bpo-39991: IPv6 address "fe80::5054:ff:fe9" looks like a MAC address - # (same string length) but must be skipped - data = '''\ -Name Mtu Network Address Ipkts Ierrs Idrop Opkts Oerrs Coll -vtnet 1500 52:54:00:9d:0e:67 10017 0 0 8174 0 0 -vtnet - fe80::%vtnet0 fe80::5054:ff:fe9 0 - - 4 - - -vtnet - 192.168.122.0 192.168.122.45 8844 - - 8171 - - -lo0 16384 lo0 260148 0 0 260148 0 0 -lo0 - ::1/128 ::1 193 - - 193 - - - ff01::1%lo0 - ff02::2:2eb7:74fa - ff02::2:ff2e:b774 - ff02::1%lo0 - ff02::1:ff00:1%lo -lo0 - fe80::%lo0/64 fe80::1%lo0 0 - - 0 - - - ff01::1%lo0 - ff02::2:2eb7:74fa - ff02::2:ff2e:b774 - ff02::1%lo0 - ff02::1:ff00:1%lo -lo0 - 127.0.0.0/8 127.0.0.1 259955 - - 259955 - - - 224.0.0.1 -''' - - with mock.patch.multiple(self.uuid, - _MAC_DELIM=b':', - _MAC_OMITS_LEADING_ZEROES=False, - _get_command_stdout=mock_get_command_stdout(data)): - mac = self.uuid._find_mac_under_heading( - command='netstat', - args='-ian', - heading=b'Address', - ) - - self.assertEqual(mac, 0x5254009d0e67) - - def test_find_mac_near_keyword(self): - # key and value are on the same line - data = ''' -fake Link encap:UNSPEC hwaddr 00-00 -cscotun0 Link encap:UNSPEC HWaddr 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00 -eth0 Link encap:Ethernet HWaddr 12:34:56:78:90:ab -''' - - # The above data will only be parsed properly on non-AIX unixes. - with mock.patch.multiple(self.uuid, - _MAC_DELIM=b':', - _MAC_OMITS_LEADING_ZEROES=False, - _get_command_stdout=mock_get_command_stdout(data)): - mac = self.uuid._find_mac_near_keyword( - command='ifconfig', - args='', - keywords=[b'hwaddr'], - get_word_index=lambda x: x + 1, - ) - - self.assertEqual(mac, 0x1234567890ab) - - def check_node(self, node, requires=None): - if requires and node is None: - self.skipTest('requires ' + requires) - hex = '%012x' % node - if support.verbose >= 2: - print(hex, end=' ') - self.assertTrue(0 < node < (1 << 48), - "%s is not an RFC 4122 node ID" % hex) - - @unittest.skipUnless(_uuid._ifconfig_getnode in _uuid._GETTERS, - "ifconfig is not used for introspection on this platform") - def test_ifconfig_getnode(self): - node = self.uuid._ifconfig_getnode() - self.check_node(node, 'ifconfig') - - @unittest.skipUnless(_uuid._ip_getnode in _uuid._GETTERS, - "ip is not used for introspection on this platform") - def test_ip_getnode(self): - node = self.uuid._ip_getnode() - self.check_node(node, 'ip') - - @unittest.skipUnless(_uuid._arp_getnode in _uuid._GETTERS, - "arp is not used for introspection on this platform") - def test_arp_getnode(self): - node = self.uuid._arp_getnode() - self.check_node(node, 'arp') - - @unittest.skipUnless(_uuid._lanscan_getnode in _uuid._GETTERS, - "lanscan is not used for introspection on this platform") - def test_lanscan_getnode(self): - node = self.uuid._lanscan_getnode() - self.check_node(node, 'lanscan') - - @unittest.skipUnless(_uuid._netstat_getnode in _uuid._GETTERS, - "netstat is not used for introspection on this platform") - def test_netstat_getnode(self): - node = self.uuid._netstat_getnode() - self.check_node(node, 'netstat') - - def test_random_getnode(self): - node = self.uuid._random_getnode() - # The multicast bit, i.e. the least significant bit of first octet, - # must be set for randomly generated MAC addresses. See RFC 4122, - # $4.1.6. - self.assertTrue(node & (1 << 40), '%012x' % node) - self.check_node(node) - - node2 = self.uuid._random_getnode() - self.assertNotEqual(node2, node, '%012x' % node) - -class TestInternalsWithoutExtModule(BaseTestInternals, unittest.TestCase): - uuid = py_uuid - -@unittest.skipUnless(c_uuid, 'requires the C _uuid module') -class TestInternalsWithExtModule(BaseTestInternals, unittest.TestCase): - uuid = c_uuid - - @unittest.skipUnless(os.name == 'posix', 'requires Posix') - def test_unix_getnode(self): - if not importable('_uuid') and not importable('ctypes'): - self.skipTest("neither _uuid extension nor ctypes available") - try: # Issues 1481, 3581: _uuid_generate_time() might be None. - node = self.uuid._unix_getnode() - except TypeError: - self.skipTest('requires uuid_generate_time') - self.check_node(node, 'unix') +# @unittest.skipUnless(c_uuid, 'requires the C _uuid module') +# class TestUUIDWithExtModule(BaseTestUUID, unittest.TestCase): +# uuid = c_uuid - @unittest.skipUnless(os.name == 'nt', 'requires Windows') - def test_windll_getnode(self): - node = self.uuid._windll_getnode() - self.check_node(node) if __name__ == '__main__': - unittest.main() + unittest.main(verbosity=2) From e5839ffe35fdab6c4c51b3e33a72007ae199b5fa Mon Sep 17 00:00:00 2001 From: stu Date: Tue, 8 Nov 2022 19:26:36 +0800 Subject: [PATCH 062/137] BuildNativeClass - make sure we fix reserved words and also ensure tp$richcompare exists if ob$eq was set --- src/abstract.js | 27 +++++++++++++++++++++------ src/str.js | 1 + 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/abstract.js b/src/abstract.js index c8df64bbce..77fd09bb20 100644 --- a/src/abstract.js +++ b/src/abstract.js @@ -1022,6 +1022,14 @@ Sk.abstr.setUpBuiltinMro = function (child) { }); }; +let fixReserved = (x) => { + if (Sk.builtin.str && Sk.builtin.str.$fixReserved) { + fixReserved = Sk.builtin.str.$fixReserved; + return Sk.builtin.str.$fixReserved(x); + } + return x; +}; + /** * @param {FunctionConstructor} klass * @param {Object=} getsets @@ -1034,7 +1042,7 @@ Sk.abstr.setUpGetSets = function (klass, getsets) { getsets = getsets || klass_proto.tp$getsets || {}; Object.entries(getsets).forEach(([getset_name, getset_def]) => { getset_def.$name = getset_name; - klass_proto[getset_name] = new Sk.builtin.getset_descriptor(klass, getset_def); + klass_proto[fixReserved(getset_name)] = new Sk.builtin.getset_descriptor(klass, getset_def); }); Object.defineProperty(klass_proto, "tp$getsets", { value: null, writable: true }); }; @@ -1052,7 +1060,7 @@ Sk.abstr.setUpMethods = function (klass, methods) { methods = methods || klass_proto.tp$methods || {}; Object.entries(methods).forEach(([method_name, method_def]) => { method_def.$name = method_name; - klass_proto[method_name] = new Sk.builtin.method_descriptor(klass, method_def); + klass_proto[fixReserved(method_name)] = new Sk.builtin.method_descriptor(klass, method_def); }); Object.defineProperty(klass_proto, "tp$methods", { value: null, writable: true }); }; @@ -1070,22 +1078,24 @@ Sk.abstr.setUpClassMethods = function (klass, methods) { methods = methods || klass_proto.tp$classmethods || {}; Object.entries(methods).forEach(([method_name, method_def]) => { method_def.$name = method_name; - klass_proto[method_name] = new Sk.builtin.classmethod_descriptor(klass, method_def); + klass_proto[fixReserved(method_name)] = new Sk.builtin.classmethod_descriptor(klass, method_def); }); Object.defineProperty(klass_proto, "tp$classmethods", { value: null, writable: true }); }; -const op2shortcut = Object.entries({ +const op2shortcut = { Eq: "ob$eq", NotEq: "ob$ne", Gt: "ob$gt", GtE: "ob$ge", Lt: "ob$lt", LtE: "ob$le", -}); +}; + +const op2shortcutEntries = Object.entries(op2shortcut); function _set_up_richcompare_wrappers(slots) { - op2shortcut.forEach(([op, shortcut]) => { + op2shortcutEntries.forEach(([op, shortcut]) => { slots[shortcut] = function (other) { return this.tp$richcompare(other, op); }; @@ -1150,6 +1160,11 @@ Sk.abstr.setUpSlots = function (klass, slots) { // Either a klass defines a tp$richcompare slot or ob$eq slots // if tp$richcompare is defined then create all the ob$eq slots _set_up_richcompare_wrappers(slots); + } else if (slots.ob$eq) { + // assume if they set one of the slots they set them all! + slots.tp$richcompare = function(other, op) { + return this[op2shortcut[op]].call(this, other); + }; } // setup some reflected slots diff --git a/src/str.js b/src/str.js index 67158437d6..8be65003c2 100755 --- a/src/str.js +++ b/src/str.js @@ -1491,3 +1491,4 @@ function fixReserved(name) { } Sk.builtin.str.reservedWords_ = reservedWords_; +Sk.builtin.str.$fixReserved = fixReserved; From 802abffafcef51ab54b71868807eefa7689ae9ac Mon Sep 17 00:00:00 2001 From: stu Date: Tue, 8 Nov 2022 19:27:31 +0800 Subject: [PATCH 063/137] Fix parsenumber which was failing in the tests for long integers that included `04e23` e.g. `0x21e12` --- src/ast.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ast.js b/src/ast.js index 43984d27a1..de17a2101b 100644 --- a/src/ast.js +++ b/src/ast.js @@ -2677,7 +2677,7 @@ function parsestrplus (c, n) { } } -const FLOAT_RE = new RegExp(Sk._tokenize.Floatnumber); +const FLOAT_RE = new RegExp("^" + Sk._tokenize.Floatnumber); const underscore = /_/g; function parsenumber(c, s, lineno) { From 25cb4792b6d621b2b57933b1e93819a7d18b3c42 Mon Sep 17 00:00:00 2001 From: stu Date: Tue, 8 Nov 2022 19:27:55 +0800 Subject: [PATCH 064/137] Partial implementation of to and from bytes with tests --- src/int.js | 110 ++++++++- test/unit3/test_bool.py | 4 + test/unit3/test_long.py | 498 ++++++++++++++++++++-------------------- 3 files changed, 359 insertions(+), 253 deletions(-) diff --git a/src/int.js b/src/int.js index 9e114f47cc..2d17ccdc86 100644 --- a/src/int.js +++ b/src/int.js @@ -50,7 +50,10 @@ Sk.builtin.int_ = Sk.abstr.buildNativeClass("int", { x = args[0]; base = Sk.builtin.none.none$; } else { - args = Sk.abstr.copyKeywordsToNamedArgs("int", [null, "base"], args, kwargs, [INT_ZERO, Sk.builtin.none.none$]); + args = Sk.abstr.copyKeywordsToNamedArgs("int", [null, "base"], args, kwargs, [ + INT_ZERO, + Sk.builtin.none.none$, + ]); x = args[0]; base = args[1]; } @@ -112,7 +115,10 @@ Sk.builtin.int_ = Sk.abstr.buildNativeClass("int", { (v, w) => v - w, (v, w) => JSBI.numberIfSafe(JSBI.subtract(v, w)) ), - nb$multiply: numberSlot((v, w) => v * w, (v, w) => v === JSBI.__ZERO || w === JSBI.__ZERO ? 0 : JSBI.multiply(v, w)), + nb$multiply: numberSlot( + (v, w) => v * w, + (v, w) => (v === JSBI.__ZERO || w === JSBI.__ZERO ? 0 : JSBI.multiply(v, w)) + ), nb$divide: trueDivide, nb$floor_divide: numberDivisionSlot((v, w) => Math.floor(v / w), BigIntFloorDivide), nb$remainder: numberDivisionSlot( @@ -213,6 +219,51 @@ Sk.builtin.int_ = Sk.abstr.buildNativeClass("int", { $doc: "the imaginary part of a complex number", }, }, + classmethods: { + from_bytes: { + $meth(args, kws) { + Sk.abstr.checkArgsLen("from_bytes", args, 0, 2); + let [bytes, byteorder, signed] = Sk.abstr.copyKeywordsToNamedArgs( + "from_bytes", + ["bytes", "byteorder", "signed"], + args, + kws, + [Sk.builtin.bool.false$] + ); + const s_lit = new Sk.builtin.str("little"); + const s_big = new Sk.builtin.str("big"); + if (byteorder !== s_big && byteorder !== s_lit) { + throw new Sk.builtin.ValueError("byteorder must be either 'little' or 'big'"); + } + if (!(bytes instanceof Sk.builtin.bytes)) { + // not quite right - we should call pyObjectBytes - which fails on integers + // but good enough for now + bytes = Sk.misceval.callsimArray(Sk.builtin.bytes, [bytes]); + } + if (Sk.misceval.isTrue(signed)) { + /** @todo - from_bytes with signed=True */ + throw new Sk.builtin.NotImplementedError( + "from_bytes with signed=True is not yet implemented in Skulpt" + ); + } + const uint8 = bytes.valueOf(); + const hex = []; + uint8.forEach((x) => { + hex.push(x.toString(16).padStart(2, "0")); + }); + if (byteorder === s_lit) { + hex.reverse(); + } + const asInt = new Sk.builtin.int_(JSBI.numberIfSafe(JSBI.BigInt("0x" + (hex.join("") || "0")))); + if (this === Sk.builtin.int_) { + return asInt; + } else { + return Sk.misceval.callsimArray(this, [asInt]); + } + }, + $flags: { FastCall: true }, + }, + }, methods: /** @lends {Sk.builtin.int_.prototype}*/ { conjugate: { $meth: cloneSelf, @@ -234,8 +285,59 @@ Sk.builtin.int_ = Sk.abstr.buildNativeClass("int", { $doc: "Number of bits necessary to represent self in binary.\n\n>>> bin(37)\n'0b100101'\n>>> (37).bit_length()\n6", }, to_bytes: { - $meth() { - throw new Sk.builtin.NotImplementedError("Not yet implemented in Skulpt"); + $meth(args, kws) { + Sk.abstr.checkArgsLen("to_bytes", args, 0, 2); + let [length, byteorder, signed] = Sk.abstr.copyKeywordsToNamedArgs( + "to_bytes", + ["length", "byteorder", "signed"], + args, + kws, + [Sk.builtin.bool.false$] + ); + const s_lit = new Sk.builtin.str("little"); + const s_big = new Sk.builtin.str("big"); + if (byteorder !== s_big && byteorder !== s_lit) { + throw new Sk.builtin.ValueError("byteorder must be either 'little' or 'big'"); + } + length = Sk.misceval.asIndexSized(length, Sk.builtin.OverflowError); + if (length < 0) { + throw new Sk.builtin.ValueError("length argument must be non-negative"); + } + if (Sk.misceval.isTrue(signed)) { + /** @todo - to_bytes with signed=True */ + throw new Sk.builtin.NotImplementedError( + "from_bytes with signed=True is not yet implemented in Skulpt" + ); + } + if (this.nb$isnegative()) { + throw new Sk.builtin.OverflowError("can't convert negative int to unsigned"); + } + let hex = JSBI.BigInt(this.v).toString(16); + if (hex.length % 2) { + hex = "0" + hex; + } + const len = hex.length / 2; + if (len > length) { + if (length === 0 && hex === "00") { + return new Sk.builtin.bytes(); + } + throw new Sk.builtin.OverflowError("int too big to convert"); + } + + const u8 = new Array(length).fill(0); + let i = length - len; + let j = 0; + + while (i < length) { + u8[i] = parseInt(hex.slice(j, j + 2), 16); + i += 1; + j += 2; + } + + if (byteorder === s_lit) { + u8.reverse(); + } + return new Sk.builtin.bytes(u8); }, $flags: { FastCall: true }, $textsig: "($self, /, length, byteorder, *, signed=False)", diff --git a/test/unit3/test_bool.py b/test/unit3/test_bool.py index 2de60de3f0..14d19ca30e 100644 --- a/test/unit3/test_bool.py +++ b/test/unit3/test_bool.py @@ -380,6 +380,10 @@ def foo(x, y): # return -1 # self.assertRaises(ValueError, bool, Eggs()) + def test_from_bytes(self): + self.assertIs(bool.from_bytes(b'\x00'*8, 'big'), False) + self.assertIs(bool.from_bytes(b'abcd', 'little'), True) + # def test_sane_len(self): # # this test just tests our assumptions about __len__ # # this will start failing if __len__ changes assertions diff --git a/test/unit3/test_long.py b/test/unit3/test_long.py index 10157f7af4..32df86d330 100644 --- a/test/unit3/test_long.py +++ b/test/unit3/test_long.py @@ -1102,256 +1102,256 @@ def test_round(self): for e in bad_exponents: self.assertRaises(TypeError, round, 3, e) - # @TODO not yet implemented in skulpt - # def test_to_bytes(self): - # def check(tests, byteorder, signed=False): - # for test, expected in tests.items(): - # try: - # self.assertEqual( - # test.to_bytes(len(expected), byteorder, signed=signed), - # expected) - # except Exception as err: - # raise AssertionError( - # "failed to convert {0} with byteorder={1} and signed={2}" - # .format(test, byteorder, signed)) from err - - # # Convert integers to signed big-endian byte arrays. - # tests1 = { - # 0: b'\x00', - # 1: b'\x01', - # -1: b'\xff', - # -127: b'\x81', - # -128: b'\x80', - # -129: b'\xff\x7f', - # 127: b'\x7f', - # 129: b'\x00\x81', - # -255: b'\xff\x01', - # -256: b'\xff\x00', - # 255: b'\x00\xff', - # 256: b'\x01\x00', - # 32767: b'\x7f\xff', - # -32768: b'\xff\x80\x00', - # 65535: b'\x00\xff\xff', - # -65536: b'\xff\x00\x00', - # -8388608: b'\x80\x00\x00' - # } - # check(tests1, 'big', signed=True) - - # # Convert integers to signed little-endian byte arrays. - # tests2 = { - # 0: b'\x00', - # 1: b'\x01', - # -1: b'\xff', - # -127: b'\x81', - # -128: b'\x80', - # -129: b'\x7f\xff', - # 127: b'\x7f', - # 129: b'\x81\x00', - # -255: b'\x01\xff', - # -256: b'\x00\xff', - # 255: b'\xff\x00', - # 256: b'\x00\x01', - # 32767: b'\xff\x7f', - # -32768: b'\x00\x80', - # 65535: b'\xff\xff\x00', - # -65536: b'\x00\x00\xff', - # -8388608: b'\x00\x00\x80' - # } - # check(tests2, 'little', signed=True) - - # # Convert integers to unsigned big-endian byte arrays. - # tests3 = { - # 0: b'\x00', - # 1: b'\x01', - # 127: b'\x7f', - # 128: b'\x80', - # 255: b'\xff', - # 256: b'\x01\x00', - # 32767: b'\x7f\xff', - # 32768: b'\x80\x00', - # 65535: b'\xff\xff', - # 65536: b'\x01\x00\x00' - # } - # check(tests3, 'big', signed=False) - - # # Convert integers to unsigned little-endian byte arrays. - # tests4 = { - # 0: b'\x00', - # 1: b'\x01', - # 127: b'\x7f', - # 128: b'\x80', - # 255: b'\xff', - # 256: b'\x00\x01', - # 32767: b'\xff\x7f', - # 32768: b'\x00\x80', - # 65535: b'\xff\xff', - # 65536: b'\x00\x00\x01' - # } - # check(tests4, 'little', signed=False) - - # self.assertRaises(OverflowError, (256).to_bytes, 1, 'big', signed=False) - # self.assertRaises(OverflowError, (256).to_bytes, 1, 'big', signed=True) - # self.assertRaises(OverflowError, (256).to_bytes, 1, 'little', signed=False) - # self.assertRaises(OverflowError, (256).to_bytes, 1, 'little', signed=True) - # self.assertRaises(OverflowError, (-1).to_bytes, 2, 'big', signed=False) - # self.assertRaises(OverflowError, (-1).to_bytes, 2, 'little', signed=False) - # self.assertEqual((0).to_bytes(0, 'big'), b'') - # self.assertEqual((1).to_bytes(5, 'big'), b'\x00\x00\x00\x00\x01') - # self.assertEqual((0).to_bytes(5, 'big'), b'\x00\x00\x00\x00\x00') - # self.assertEqual((-1).to_bytes(5, 'big', signed=True), - # b'\xff\xff\xff\xff\xff') - # self.assertRaises(OverflowError, (1).to_bytes, 0, 'big') - - # @TODO not yet implemented in skulpt - # def test_from_bytes(self): - # def check(tests, byteorder, signed=False): - # for test, expected in tests.items(): - # try: - # self.assertEqual( - # int.from_bytes(test, byteorder, signed=signed), - # expected) - # except Exception as err: - # raise AssertionError( - # "failed to convert {0} with byteorder={1!r} and signed={2}" - # .format(test, byteorder, signed)) from err - - # # Convert signed big-endian byte arrays to integers. - # tests1 = { - # b'': 0, - # b'\x00': 0, - # b'\x00\x00': 0, - # b'\x01': 1, - # b'\x00\x01': 1, - # b'\xff': -1, - # b'\xff\xff': -1, - # b'\x81': -127, - # b'\x80': -128, - # b'\xff\x7f': -129, - # b'\x7f': 127, - # b'\x00\x81': 129, - # b'\xff\x01': -255, - # b'\xff\x00': -256, - # b'\x00\xff': 255, - # b'\x01\x00': 256, - # b'\x7f\xff': 32767, - # b'\x80\x00': -32768, - # b'\x00\xff\xff': 65535, - # b'\xff\x00\x00': -65536, - # b'\x80\x00\x00': -8388608 - # } - # check(tests1, 'big', signed=True) - - # # Convert signed little-endian byte arrays to integers. - # tests2 = { - # b'': 0, - # b'\x00': 0, - # b'\x00\x00': 0, - # b'\x01': 1, - # b'\x00\x01': 256, - # b'\xff': -1, - # b'\xff\xff': -1, - # b'\x81': -127, - # b'\x80': -128, - # b'\x7f\xff': -129, - # b'\x7f': 127, - # b'\x81\x00': 129, - # b'\x01\xff': -255, - # b'\x00\xff': -256, - # b'\xff\x00': 255, - # b'\x00\x01': 256, - # b'\xff\x7f': 32767, - # b'\x00\x80': -32768, - # b'\xff\xff\x00': 65535, - # b'\x00\x00\xff': -65536, - # b'\x00\x00\x80': -8388608 - # } - # check(tests2, 'little', signed=True) - - # # Convert unsigned big-endian byte arrays to integers. - # tests3 = { - # b'': 0, - # b'\x00': 0, - # b'\x01': 1, - # b'\x7f': 127, - # b'\x80': 128, - # b'\xff': 255, - # b'\x01\x00': 256, - # b'\x7f\xff': 32767, - # b'\x80\x00': 32768, - # b'\xff\xff': 65535, - # b'\x01\x00\x00': 65536, - # } - # check(tests3, 'big', signed=False) - - # # Convert integers to unsigned little-endian byte arrays. - # tests4 = { - # b'': 0, - # b'\x00': 0, - # b'\x01': 1, - # b'\x7f': 127, - # b'\x80': 128, - # b'\xff': 255, - # b'\x00\x01': 256, - # b'\xff\x7f': 32767, - # b'\x00\x80': 32768, - # b'\xff\xff': 65535, - # b'\x00\x00\x01': 65536, - # } - # check(tests4, 'little', signed=False) - - # class myint(int): - # pass + # @TODO signed=True not yet implemented in skulpt + def test_to_bytes(self): + def check(tests, byteorder, signed=False): + for test, expected in tests.items(): + try: + self.assertEqual( + test.to_bytes(len(expected), byteorder, signed=signed), + expected) + except Exception as err: + raise AssertionError( + "failed to convert {0} with byteorder={1} and signed={2}" + .format(test, byteorder, signed)) from err + + # Convert integers to signed big-endian byte arrays. + tests1 = { + 0: b'\x00', + 1: b'\x01', + -1: b'\xff', + -127: b'\x81', + -128: b'\x80', + -129: b'\xff\x7f', + 127: b'\x7f', + 129: b'\x00\x81', + -255: b'\xff\x01', + -256: b'\xff\x00', + 255: b'\x00\xff', + 256: b'\x01\x00', + 32767: b'\x7f\xff', + -32768: b'\xff\x80\x00', + 65535: b'\x00\xff\xff', + -65536: b'\xff\x00\x00', + -8388608: b'\x80\x00\x00' + } + # check(tests1, 'big', signed=True) + + # Convert integers to signed little-endian byte arrays. + tests2 = { + 0: b'\x00', + 1: b'\x01', + -1: b'\xff', + -127: b'\x81', + -128: b'\x80', + -129: b'\x7f\xff', + 127: b'\x7f', + 129: b'\x81\x00', + -255: b'\x01\xff', + -256: b'\x00\xff', + 255: b'\xff\x00', + 256: b'\x00\x01', + 32767: b'\xff\x7f', + -32768: b'\x00\x80', + 65535: b'\xff\xff\x00', + -65536: b'\x00\x00\xff', + -8388608: b'\x00\x00\x80' + } + # check(tests2, 'little', signed=True) + + # Convert integers to unsigned big-endian byte arrays. + tests3 = { + 0: b'\x00', + 1: b'\x01', + 127: b'\x7f', + 128: b'\x80', + 255: b'\xff', + 256: b'\x01\x00', + 32767: b'\x7f\xff', + 32768: b'\x80\x00', + 65535: b'\xff\xff', + 65536: b'\x01\x00\x00' + } + check(tests3, 'big', signed=False) + + # Convert integers to unsigned little-endian byte arrays. + tests4 = { + 0: b'\x00', + 1: b'\x01', + 127: b'\x7f', + 128: b'\x80', + 255: b'\xff', + 256: b'\x00\x01', + 32767: b'\xff\x7f', + 32768: b'\x00\x80', + 65535: b'\xff\xff', + 65536: b'\x00\x00\x01' + } + check(tests4, 'little', signed=False) + + self.assertRaises(OverflowError, (256).to_bytes, 1, 'big', signed=False) + # self.assertRaises(OverflowError, (256).to_bytes, 1, 'big', signed=True) + self.assertRaises(OverflowError, (256).to_bytes, 1, 'little', signed=False) + # self.assertRaises(OverflowError, (256).to_bytes, 1, 'little', signed=True) + self.assertRaises(OverflowError, (-1).to_bytes, 2, 'big', signed=False) + self.assertRaises(OverflowError, (-1).to_bytes, 2, 'little', signed=False) + self.assertEqual((0).to_bytes(0, 'big'), b'') + self.assertEqual((1).to_bytes(5, 'big'), b'\x00\x00\x00\x00\x01') + self.assertEqual((0).to_bytes(5, 'big'), b'\x00\x00\x00\x00\x00') + # self.assertEqual((-1).to_bytes(5, 'big', signed=True), + # b'\xff\xff\xff\xff\xff') + self.assertRaises(OverflowError, (1).to_bytes, 0, 'big') + + # @TODO signed=True not yet implemented in skulpt + def test_from_bytes(self): + def check(tests, byteorder, signed=False): + for test, expected in tests.items(): + try: + self.assertEqual( + int.from_bytes(test, byteorder, signed=signed), + expected) + except Exception as err: + raise AssertionError( + "failed to convert {0} with byteorder={1!r} and signed={2}" + .format(test, byteorder, signed)) from err + + # Convert signed big-endian byte arrays to integers. + tests1 = { + b'': 0, + b'\x00': 0, + b'\x00\x00': 0, + b'\x01': 1, + b'\x00\x01': 1, + b'\xff': -1, + b'\xff\xff': -1, + b'\x81': -127, + b'\x80': -128, + b'\xff\x7f': -129, + b'\x7f': 127, + b'\x00\x81': 129, + b'\xff\x01': -255, + b'\xff\x00': -256, + b'\x00\xff': 255, + b'\x01\x00': 256, + b'\x7f\xff': 32767, + b'\x80\x00': -32768, + b'\x00\xff\xff': 65535, + b'\xff\x00\x00': -65536, + b'\x80\x00\x00': -8388608 + } + # check(tests1, 'big', signed=True) + + # Convert signed little-endian byte arrays to integers. + tests2 = { + b'': 0, + b'\x00': 0, + b'\x00\x00': 0, + b'\x01': 1, + b'\x00\x01': 256, + b'\xff': -1, + b'\xff\xff': -1, + b'\x81': -127, + b'\x80': -128, + b'\x7f\xff': -129, + b'\x7f': 127, + b'\x81\x00': 129, + b'\x01\xff': -255, + b'\x00\xff': -256, + b'\xff\x00': 255, + b'\x00\x01': 256, + b'\xff\x7f': 32767, + b'\x00\x80': -32768, + b'\xff\xff\x00': 65535, + b'\x00\x00\xff': -65536, + b'\x00\x00\x80': -8388608 + } + # check(tests2, 'little', signed=True) + + # Convert unsigned big-endian byte arrays to integers. + tests3 = { + b'': 0, + b'\x00': 0, + b'\x01': 1, + b'\x7f': 127, + b'\x80': 128, + b'\xff': 255, + b'\x01\x00': 256, + b'\x7f\xff': 32767, + b'\x80\x00': 32768, + b'\xff\xff': 65535, + b'\x01\x00\x00': 65536, + } + check(tests3, 'big', signed=False) + + # Convert integers to unsigned little-endian byte arrays. + tests4 = { + b'': 0, + b'\x00': 0, + b'\x01': 1, + b'\x7f': 127, + b'\x80': 128, + b'\xff': 255, + b'\x00\x01': 256, + b'\xff\x7f': 32767, + b'\x00\x80': 32768, + b'\xff\xff': 65535, + b'\x00\x00\x01': 65536, + } + check(tests4, 'little', signed=False) + + class myint(int): + pass + + self.assertIs(type(myint.from_bytes(b'\x00', 'big')), myint) + self.assertEqual(myint.from_bytes(b'\x01', 'big'), 1) + self.assertIs( + type(myint.from_bytes(b'\x00', 'big', signed=False)), myint) + self.assertEqual(myint.from_bytes(b'\x01', 'big', signed=False), 1) + self.assertIs(type(myint.from_bytes(b'\x00', 'little')), myint) + self.assertEqual(myint.from_bytes(b'\x01', 'little'), 1) + self.assertIs(type(myint.from_bytes( + b'\x00', 'little', signed=False)), myint) + self.assertEqual(myint.from_bytes(b'\x01', 'little', signed=False), 1) + # self.assertEqual( + # int.from_bytes([255, 0, 0], 'big', signed=True), -65536) + # self.assertEqual( + # int.from_bytes((255, 0, 0), 'big', signed=True), -65536) + # self.assertEqual(int.from_bytes( + # bytearray(b'\xff\x00\x00'), 'big', signed=True), -65536) + # self.assertEqual(int.from_bytes( + # bytearray(b'\xff\x00\x00'), 'big', signed=True), -65536) + # self.assertEqual(int.from_bytes( + # array.array('B', b'\xff\x00\x00'), 'big', signed=True), -65536) + # self.assertEqual(int.from_bytes( + # memoryview(b'\xff\x00\x00'), 'big', signed=True), -65536) + self.assertRaises(ValueError, int.from_bytes, [256], 'big') + self.assertRaises(ValueError, int.from_bytes, [0], 'big\x00') + self.assertRaises(ValueError, int.from_bytes, [0], 'little\x00') + self.assertRaises(TypeError, int.from_bytes, "", 'big') + self.assertRaises(TypeError, int.from_bytes, "\x00", 'big') + # self.assertRaises(TypeError, int.from_bytes, 0, 'big') + # self.assertRaises(TypeError, int.from_bytes, 0, 'big', True) + self.assertRaises(TypeError, myint.from_bytes, "", 'big') + self.assertRaises(TypeError, myint.from_bytes, "\x00", 'big') + # self.assertRaises(TypeError, myint.from_bytes, 0, 'big') + # self.assertRaises(TypeError, int.from_bytes, 0, 'big', True) + + class myint2(int): + def __new__(cls, value): + return int.__new__(cls, value + 1) + + i = myint2.from_bytes(b'\x01', 'big') + self.assertIs(type(i), myint2) + self.assertEqual(i, 2) + + class myint3(int): + def __init__(self, value): + self.foo = 'bar' - # self.assertIs(type(myint.from_bytes(b'\x00', 'big')), myint) - # self.assertEqual(myint.from_bytes(b'\x01', 'big'), 1) - # self.assertIs( - # type(myint.from_bytes(b'\x00', 'big', signed=False)), myint) - # self.assertEqual(myint.from_bytes(b'\x01', 'big', signed=False), 1) - # self.assertIs(type(myint.from_bytes(b'\x00', 'little')), myint) - # self.assertEqual(myint.from_bytes(b'\x01', 'little'), 1) - # self.assertIs(type(myint.from_bytes( - # b'\x00', 'little', signed=False)), myint) - # self.assertEqual(myint.from_bytes(b'\x01', 'little', signed=False), 1) - # self.assertEqual( - # int.from_bytes([255, 0, 0], 'big', signed=True), -65536) - # self.assertEqual( - # int.from_bytes((255, 0, 0), 'big', signed=True), -65536) - # self.assertEqual(int.from_bytes( - # bytearray(b'\xff\x00\x00'), 'big', signed=True), -65536) - # self.assertEqual(int.from_bytes( - # bytearray(b'\xff\x00\x00'), 'big', signed=True), -65536) - # self.assertEqual(int.from_bytes( - # array.array('B', b'\xff\x00\x00'), 'big', signed=True), -65536) - # self.assertEqual(int.from_bytes( - # memoryview(b'\xff\x00\x00'), 'big', signed=True), -65536) - # self.assertRaises(ValueError, int.from_bytes, [256], 'big') - # self.assertRaises(ValueError, int.from_bytes, [0], 'big\x00') - # self.assertRaises(ValueError, int.from_bytes, [0], 'little\x00') - # self.assertRaises(TypeError, int.from_bytes, "", 'big') - # self.assertRaises(TypeError, int.from_bytes, "\x00", 'big') - # self.assertRaises(TypeError, int.from_bytes, 0, 'big') - # self.assertRaises(TypeError, int.from_bytes, 0, 'big', True) - # self.assertRaises(TypeError, myint.from_bytes, "", 'big') - # self.assertRaises(TypeError, myint.from_bytes, "\x00", 'big') - # self.assertRaises(TypeError, myint.from_bytes, 0, 'big') - # self.assertRaises(TypeError, int.from_bytes, 0, 'big', True) - - # class myint2(int): - # def __new__(cls, value): - # return int.__new__(cls, value + 1) - - # i = myint2.from_bytes(b'\x01', 'big') - # self.assertIs(type(i), myint2) - # self.assertEqual(i, 2) - - # class myint3(int): - # def __init__(self, value): - # self.foo = 'bar' - - # i = myint3.from_bytes(b'\x01', 'big') - # self.assertIs(type(i), myint3) - # self.assertEqual(i, 1) - # self.assertEqual(getattr(i, 'foo', 'none'), 'bar') + i = myint3.from_bytes(b'\x01', 'big') + self.assertIs(type(i), myint3) + self.assertEqual(i, 1) + self.assertEqual(getattr(i, 'foo', 'none'), 'bar') def test_access_to_nonexistent_digit_0(self): # http://bugs.python.org/issue14630: A bug in _PyLong_Copy meant that From a18083df21a3090573fef770f0ec4d94590af0b1 Mon Sep 17 00:00:00 2001 From: stu Date: Tue, 8 Nov 2022 19:30:03 +0800 Subject: [PATCH 065/137] Update uuid.js --- src/lib/uuid.js | 97 ++++++++++++++++---------------- src/lib/uuid_.js | 141 ----------------------------------------------- 2 files changed, 48 insertions(+), 190 deletions(-) delete mode 100644 src/lib/uuid_.js diff --git a/src/lib/uuid.js b/src/lib/uuid.js index fe4fd2c40d..a1d273729f 100644 --- a/src/lib/uuid.js +++ b/src/lib/uuid.js @@ -3,20 +3,22 @@ function $builtinmodule() { builtin: { bytes: pyBytes, str: pyStr, - int: pyInt, + int_: pyInt, TypeError: pyTypeError, ValueError: pyValueError, NotImplementedError: pyNotImplementedError, none: { none$: pyNone }, NotImplemented: { NotImplemented$: pyNotImplemented }, - len: pyLen, }, - abstr: { buildNativeClass, checkArgsLen, copyKeywordsToNamedArgs }, - misceval: { pyCall, richCompareBool }, + abstr: { buildNativeClass, checkArgsLen, copyKeywordsToNamedArgs, setUpModuleMethods }, + misceval: { callsimArray: pyCall, richCompareBool }, } = Sk; const mod = { __name__: new pyStr("uuid"), + RESERVED_NCS: pyNone, + RFC_4122: pyNone, + RESERVED_FUTURE: pyNone, }; const fromBytes = pyInt.tp$getattr(new pyStr("from_bytes")); @@ -32,7 +34,7 @@ function $builtinmodule() { const lt = (a, b) => richCompareBool(a, b, "Lt"); const ge = (a, b) => richCompareBool(a, b, "GtE"); - function notImplemneted() { + function notImplemented() { throw new pyNotImplementedError("Not yet implemneted in Skulpt"); } @@ -43,7 +45,7 @@ function $builtinmodule() { checkArgsLen("UUID", args, 0, 6); let [hex, bytes, bytes_le, fields, int, version, is_safe] = copyKeywordsToNamedArgs( "UUID", - ["hex", "bytes", "bytes_le", "fields", , "version", "is_safe"], + ["hex", "bytes", "bytes_le", "fields", "int" , "version", "is_safe"], args, kws, [pyNone, pyNone, pyNone, pyNone, pyNone, pyNone, pyNone] @@ -57,10 +59,10 @@ function $builtinmodule() { hex = hex.toString().replace("urn:", "").replace("uuid:", ""); let start = 0, end = hex.length - 1; - while ("{}".indexOf(hex[start] >= 0)) { + while ("{}".indexOf(hex[start]) >= 0) { start++; } - while ("{}".indexOf(hex[end] >= 0)) { + while ("{}".indexOf(hex[end]) >= 0) { end--; } hex = hex.slice(start, end + 1); @@ -68,6 +70,17 @@ function $builtinmodule() { if (hex.length !== 32) { throw new pyValueError("badly formed hexadecimal UUID string"); } + int = pyCall(pyInt, [new pyStr(hex), _16]); + } + + if (bytes_le !== pyNone) { + if (!(bytes_le instanceof pyBytes)) { + throw new pyTypeError("bytes_le should be a bytes instance"); + } + bytes_le = bytes_le.valueOf(); + if (bytes_le.length !== 16) { + throw new pyValueError("bytes_le is not a 16-char string"); + } bytes = [ bytes_le[3], bytes_le[2], @@ -81,27 +94,18 @@ function $builtinmodule() { bytes.push(...bytes_le.slice(8)); bytes = new pyBytes(bytes); } - - if (bytes_le !== pyNone) { - if (!(bytes_le instanceof pyBytes)) { - throw new pyTypeError("bytes_le should be a bytes instance"); - } - bytes_le = bytes_le.valueOf(); - if (bytes_le.length !== 16) { - throw new pyValueError("bytes_le is not a 16-char string"); - } - bytes = new pyBytes(); - } if (bytes !== pyNone) { if (!(bytes instanceof pyBytes)) { throw new pyTypeError("bytes_le should be a bytes instance"); } - if (!bytes.valueOf().length !== 16) { + if (bytes.valueOf().length !== 16) { throw new pyValueError("bytes is not a 16-char string"); } int = pyCall(fromBytes, [bytes], ["byteorder", _s_big]); } - + if (fields !== pyNone) { + throw new pyNotImplementedError("fields argument is not yet supported"); + } if (int !== pyNone) { if (lt(int, _0) || ge(int, _intMax)) { throw new pyValueError("int is out of range (need a 128-bit value)"); @@ -129,18 +133,13 @@ function $builtinmodule() { if (!(other instanceof UUID)) { return pyNotImplemented; } - return this.$int.tp$richcompare(other, op); + return this.$int.tp$richcompare(other.$int, op); }, + tp$as_number: true, + nb$int() { + return this.$int; + } }, - methods: { - __int__: { - $meth() { - return this.$int; - }, - $flags: { NoArgs: true }, - }, - }, - getsets: { int: { $get() { @@ -167,67 +166,67 @@ function $builtinmodule() { }, fields: { $get() { - return notImplemneted(); + return notImplemented(); }, }, time_low: { $get() { - return notImplemneted(); + return notImplemented(); }, }, time_mid: { $get() { - return notImplemneted(); + return notImplemented(); }, }, time_hi_version: { $get() { - return notImplemneted(); + return notImplemented(); }, }, clock_seq_hi_variant: { $get() { - return notImplemneted(); + return notImplemented(); }, }, clock_seq_low: { $get() { - return notImplemneted(); + return notImplemented(); }, }, time: { $get() { - return notImplemneted(); + return notImplemented(); }, }, clock_seq: { $get() { - return notImplemneted(); + return notImplemented(); }, }, node: { $get() { - return notImplemneted(); + return notImplemented(); }, }, hex: { $get() { - return notImplemneted(); + return _s_32bit.nb$remainder(this.$int); }, }, urn: { $get() { - return notImplemneted(); + return new pyStr(`urn:uuid:${this}`); }, }, variant: { $get() { - return notImplemneted(); + return notImplemented(); }, }, version: { $get() { - return notImplemneted(); + return notImplemented(); }, }, }, @@ -236,19 +235,19 @@ function $builtinmodule() { setUpModuleMethods("uuid", mod, { uuid1: { $meth() { - notImplemneted(); + notImplemented(); }, $flags: { FastCall: true }, }, uuid2: { $meth() { - notImplemneted(); + notImplemented(); }, $flags: { FastCall: true }, }, uuid3: { $meth() { - notImplemneted(); + notImplemented(); }, $flags: { FastCall: true }, }, @@ -259,9 +258,9 @@ function $builtinmodule() { }, $flags: { NoArgs: true }, }, - uuid4: { + uuid5: { $meth() { - notImplemneted(); + notImplemented(); }, $flags: { FastCall: true }, }, diff --git a/src/lib/uuid_.js b/src/lib/uuid_.js deleted file mode 100644 index 4d2704249d..0000000000 --- a/src/lib/uuid_.js +++ /dev/null @@ -1,141 +0,0 @@ -function $builtinmodule() { - const uuid = { - __name__: new pyStr("uuid"), - }; - - // Unique ID creation requires a high quality random # generator. In the browser we therefore - // require the crypto API and do not support built-in fallback to lower quality random number - // generators (like Math.random()). - - let getRandomValues; - - const rnds8 = new Uint8Array(16); - - function rng() { - // lazy load so that environments that need to polyfill have a chance to do so - if (!getRandomValues) { - // getRandomValues needs to be invoked in a context where "this" is a Crypto implementation. Also, - // find the complete implementation of crypto (msCrypto) on IE11. - getRandomValues = - (typeof crypto !== "undefined" && crypto.getRandomValues && crypto.getRandomValues.bind(crypto)) || - (typeof msCrypto !== "undefined" && - typeof msCrypto.getRandomValues === "function" && - msCrypto.getRandomValues.bind(msCrypto)); - } - return getRandomValues(rnds8); - } - - const byteToHex = []; - - for (let i = 0; i < 256; ++i) { - byteToHex.push((i + 0x100).toString(16).substr(1)); - } - - function stringify(arr, offset = 0) { - // Note: Be careful editing this code! It's been tuned for performance - // and works in ways you may not expect. See https://github.com/uuidjs/uuid/pull/434 - const uuid = ( - byteToHex[arr[offset + 0]] + - byteToHex[arr[offset + 1]] + - byteToHex[arr[offset + 2]] + - byteToHex[arr[offset + 3]] + - "-" + - byteToHex[arr[offset + 4]] + - byteToHex[arr[offset + 5]] + - "-" + - byteToHex[arr[offset + 6]] + - byteToHex[arr[offset + 7]] + - "-" + - byteToHex[arr[offset + 8]] + - byteToHex[arr[offset + 9]] + - "-" + - byteToHex[arr[offset + 10]] + - byteToHex[arr[offset + 11]] + - byteToHex[arr[offset + 12]] + - byteToHex[arr[offset + 13]] + - byteToHex[arr[offset + 14]] + - byteToHex[arr[offset + 15]] - ).toLowerCase(); - - // Consistency check for valid UUID. If this throws, it's likely due to one - // of the following: - // - One or more input array values don't map to a hex octet (leading to - // "undefined" in the uuid) - // - Invalid input values for the RFC `version` or `variant` fields - if (!validate(uuid)) { - throw TypeError("Stringified UUID is invalid"); - } - - return uuid; - } - - function v35(name, version, hashfunc) { - function generateUUID(value, namespace, buf, offset) { - if (typeof value === "string") { - value = stringToBytes(value); - } - - if (typeof namespace === "string") { - namespace = parse(namespace); - } - - if (namespace.length !== 16) { - throw TypeError("Namespace must be array-like (16 iterable integer values, 0-255)"); - } - - // Compute hash of namespace and value, Per 4.3 - // Future: Use spread syntax when supported on all platforms, e.g. `bytes = - // hashfunc([...namespace, ... value])` - let bytes = new Uint8Array(16 + value.length); - bytes.set(namespace); - bytes.set(value, namespace.length); - bytes = hashfunc(bytes); - - bytes[6] = (bytes[6] & 0x0f) | version; - bytes[8] = (bytes[8] & 0x3f) | 0x80; - - return bytes; - } - // Function#name is not settable on some platforms (#270) - try { - generateUUID.name = name; - // eslint-disable-next-line no-empty - } catch (err) {} - return generateUUID; - } - - import v35 from './v35.js'; -import sha1 from './sha1.js'; - -const v5 = v35('v5', 0x50, sha1); - - function v4() { - const rnds = rnt(); - // Per 4.4, set bits for version and `clock_seq_hi_and_reserved` - rnds[6] = (rnds[6] & 0x0f) | 0x40; - rnds[8] = (rnds[8] & 0x3f) | 0x80; - } - - uuid.UUID = buildNativeClass("uuid.UUID", { - constructor: function UUID() {}, - slots: { - tp$str() { - return new pyStr(stringify(this.$b)); - }, - }, - getsets: { - bytes: {}, - int: {}, - hex: {}, - }, - }); - - setUpModuleMethods("uuid", uuid, { - uuid4: { - $meth() { - const bytes = v4(); - return new uuid.UUID(bytes); - }, - }, - }); -} From f40336ef355d8421cffa2adb927d9b4681694a97 Mon Sep 17 00:00:00 2001 From: stu Date: Tue, 8 Nov 2022 20:21:27 +0800 Subject: [PATCH 066/137] =?UTF-8?q?Skip=20the=20uuid=20tests=20since=20nod?= =?UTF-8?q?e=20doesn't=20suppor=20the=20crypto=20API=20we=20need=20?= =?UTF-8?q?=F0=9F=98=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/unit3/test_uuid.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/unit3/test_uuid.py b/test/unit3/test_uuid.py index 90d5acfc3e..fa892556ca 100644 --- a/test/unit3/test_uuid.py +++ b/test/unit3/test_uuid.py @@ -687,4 +687,6 @@ class TestUUIDWithoutExtModule(BaseTestUUID, unittest.TestCase): if __name__ == '__main__': - unittest.main(verbosity=2) + # we can't run this since running in node doesn't support the crypto API we need + # unittest.main() + pass From ddd24e3aac228efd0b6b09669b3fae2b2969a4c1 Mon Sep 17 00:00:00 2001 From: stu Date: Tue, 8 Nov 2022 20:52:35 +0800 Subject: [PATCH 067/137] Add a polyfill for crypto.getRandomValues so we can pass the tests when running in Node --- src/lib/uuid.js | 15 +++++++++++++++ test/unit3/test_uuid.py | 4 +--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/lib/uuid.js b/src/lib/uuid.js index a1d273729f..0738ccd352 100644 --- a/src/lib/uuid.js +++ b/src/lib/uuid.js @@ -21,6 +21,21 @@ function $builtinmodule() { RESERVED_FUTURE: pyNone, }; + let crypto = Sk.global.crypto; + + if (typeof crypto === "undefined") { + // polyfill for node so the tests work + crypto = { + getRandomValues(u8) { + let l = u8.length; + while (l--) { + u8[l] = Math.floor(Math.random() * 256); + } + return u8; + }, + }; + } + const fromBytes = pyInt.tp$getattr(new pyStr("from_bytes")); const toBytes = pyInt.tp$getattr(new pyStr("to_bytes")); const _intMax = new pyInt(1).nb$lshift(new pyInt(128)); diff --git a/test/unit3/test_uuid.py b/test/unit3/test_uuid.py index fa892556ca..6c2ab59b1e 100644 --- a/test/unit3/test_uuid.py +++ b/test/unit3/test_uuid.py @@ -687,6 +687,4 @@ class TestUUIDWithoutExtModule(BaseTestUUID, unittest.TestCase): if __name__ == '__main__': - # we can't run this since running in node doesn't support the crypto API we need - # unittest.main() - pass + unittest.main() From 0afbd1497ab1b9bab2767b71a1d832e6fdd42842 Mon Sep 17 00:00:00 2001 From: stu Date: Tue, 8 Nov 2022 21:01:47 +0800 Subject: [PATCH 068/137] fix node doesn't have replaceAll polyfilled --- src/lib/uuid.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/uuid.js b/src/lib/uuid.js index 0738ccd352..a036d3c972 100644 --- a/src/lib/uuid.js +++ b/src/lib/uuid.js @@ -81,7 +81,7 @@ function $builtinmodule() { end--; } hex = hex.slice(start, end + 1); - hex = hex.replaceAll("-", ""); + hex = hex.replace(/-/g, ""); if (hex.length !== 32) { throw new pyValueError("badly formed hexadecimal UUID string"); } From 377902ddf63d734ff0b8726dd5b2cc717fa4870e Mon Sep 17 00:00:00 2001 From: stu Date: Wed, 9 Nov 2022 12:58:58 +0800 Subject: [PATCH 069/137] uuid - fix repr and minor tweaks --- src/lib/uuid.js | 51 ++++++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/src/lib/uuid.js b/src/lib/uuid.js index a036d3c972..510cb60a8b 100644 --- a/src/lib/uuid.js +++ b/src/lib/uuid.js @@ -10,8 +10,8 @@ function $builtinmodule() { none: { none$: pyNone }, NotImplemented: { NotImplemented$: pyNotImplemented }, }, - abstr: { buildNativeClass, checkArgsLen, copyKeywordsToNamedArgs, setUpModuleMethods }, - misceval: { callsimArray: pyCall, richCompareBool }, + abstr: { buildNativeClass, checkArgsLen, copyKeywordsToNamedArgs, lookupSpecial, setUpModuleMethods }, + misceval: { callsimArray: pyCall, objectRepr, richCompareBool }, } = Sk; const mod = { @@ -22,7 +22,7 @@ function $builtinmodule() { }; let crypto = Sk.global.crypto; - + if (typeof crypto === "undefined") { // polyfill for node so the tests work crypto = { @@ -46,6 +46,8 @@ function $builtinmodule() { const _s_big = new pyStr("big"); const _s_32bit = new pyStr("%032x"); + const _r_dash = /-/g; + const lt = (a, b) => richCompareBool(a, b, "Lt"); const ge = (a, b) => richCompareBool(a, b, "GtE"); @@ -53,6 +55,19 @@ function $builtinmodule() { throw new pyNotImplementedError("Not yet implemneted in Skulpt"); } + function switchBytesBytesLe(b) { + const t = new Uint8Array(b); + t[0] = b[3]; + t[1] = b[2]; + t[2] = b[1]; + t[3] = b[0]; + t[4] = b[5]; + t[5] = b[4]; + t[6] = b[7]; + t[7] = b[6]; + return t; + } + const UUID = (mod.UUID = buildNativeClass("uuid.UUID", { constructor: function () {}, slots: { @@ -60,7 +75,7 @@ function $builtinmodule() { checkArgsLen("UUID", args, 0, 6); let [hex, bytes, bytes_le, fields, int, version, is_safe] = copyKeywordsToNamedArgs( "UUID", - ["hex", "bytes", "bytes_le", "fields", "int" , "version", "is_safe"], + ["hex", "bytes", "bytes_le", "fields", "int", "version", "is_safe"], args, kws, [pyNone, pyNone, pyNone, pyNone, pyNone, pyNone, pyNone] @@ -81,7 +96,7 @@ function $builtinmodule() { end--; } hex = hex.slice(start, end + 1); - hex = hex.replace(/-/g, ""); + hex = hex.replace(_r_dash, ""); if (hex.length !== 32) { throw new pyValueError("badly formed hexadecimal UUID string"); } @@ -96,17 +111,7 @@ function $builtinmodule() { if (bytes_le.length !== 16) { throw new pyValueError("bytes_le is not a 16-char string"); } - bytes = [ - bytes_le[3], - bytes_le[2], - bytes_le[1], - bytes_le[0], - bytes_le[5], - bytes_le[4], - bytes_le[7], - bytes_le[6], - ]; - bytes.push(...bytes_le.slice(8)); + bytes = switchBytesBytesLe(bytes_le); bytes = new pyBytes(bytes); } if (bytes !== pyNone) { @@ -136,10 +141,10 @@ function $builtinmodule() { `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}` ); }, - tp$repr() { - const name = this.tp$name; - const s = new pyStr(this); - return new pyStr(`${name}(${s})`); + $r() { + const name = lookupSpecial(this.ob$type, pyStr.$name); + const str = objectRepr(this.tp$str()); + return new pyStr(`${name}(${str})`); }, tp$hash() { return this.$int.tp$hash(); @@ -153,7 +158,7 @@ function $builtinmodule() { tp$as_number: true, nb$int() { return this.$int; - } + }, }, getsets: { int: { @@ -174,9 +179,7 @@ function $builtinmodule() { bytes_le: { $get() { const bytes = this.tp$getattr(new pyStr("bytes")).valueOf(); - const bytes_le = [bytes[3], bytes[2], bytes[1], bytes[0], bytes[5], bytes[4], bytes[7], bytes[6]]; - bytes_le.push(...bytes.slice(8)); - return new pyBytes(bytes_le); + return new pyBytes(switchBytesBytesLe(bytes)); }, }, fields: { From 1b8c791ee51ac77037713dc6267c94199437ca6a Mon Sep 17 00:00:00 2001 From: stu Date: Thu, 10 Nov 2022 21:50:24 +0800 Subject: [PATCH 070/137] int: don't rely on interned strings in from_bytes, to_bytes implementation. ast.js - do full match on FLOAT_RE --- src/ast.js | 2 +- src/int.js | 31 +++++++++++++++++++++---------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/ast.js b/src/ast.js index de17a2101b..9ab77742b4 100644 --- a/src/ast.js +++ b/src/ast.js @@ -2677,7 +2677,7 @@ function parsestrplus (c, n) { } } -const FLOAT_RE = new RegExp("^" + Sk._tokenize.Floatnumber); +const FLOAT_RE = new RegExp("^" + Sk._tokenize.Floatnumber + "$"); const underscore = /_/g; function parsenumber(c, s, lineno) { diff --git a/src/int.js b/src/int.js index 2d17ccdc86..2baf45477a 100644 --- a/src/int.js +++ b/src/int.js @@ -230,11 +230,15 @@ Sk.builtin.int_ = Sk.abstr.buildNativeClass("int", { kws, [Sk.builtin.bool.false$] ); - const s_lit = new Sk.builtin.str("little"); - const s_big = new Sk.builtin.str("big"); - if (byteorder !== s_big && byteorder !== s_lit) { + let littleEndian; + if (S_LITTLE.ob$eq(byteorder)) { + littleEndian = 1; + } else if (S_BIG.ob$eq(byteorder)) { + littleEndian = 0; + } else { throw new Sk.builtin.ValueError("byteorder must be either 'little' or 'big'"); } + if (!(bytes instanceof Sk.builtin.bytes)) { // not quite right - we should call pyObjectBytes - which fails on integers // but good enough for now @@ -251,7 +255,7 @@ Sk.builtin.int_ = Sk.abstr.buildNativeClass("int", { uint8.forEach((x) => { hex.push(x.toString(16).padStart(2, "0")); }); - if (byteorder === s_lit) { + if (littleEndian) { hex.reverse(); } const asInt = new Sk.builtin.int_(JSBI.numberIfSafe(JSBI.BigInt("0x" + (hex.join("") || "0")))); @@ -294,11 +298,15 @@ Sk.builtin.int_ = Sk.abstr.buildNativeClass("int", { kws, [Sk.builtin.bool.false$] ); - const s_lit = new Sk.builtin.str("little"); - const s_big = new Sk.builtin.str("big"); - if (byteorder !== s_big && byteorder !== s_lit) { + let littleEndian; + if (S_LITTLE.ob$eq(byteorder)) { + littleEndian = 1; + } else if (S_BIG.ob$eq(byteorder)) { + littleEndian = 0; + } else { throw new Sk.builtin.ValueError("byteorder must be either 'little' or 'big'"); } + length = Sk.misceval.asIndexSized(length, Sk.builtin.OverflowError); if (length < 0) { throw new Sk.builtin.ValueError("length argument must be non-negative"); @@ -306,7 +314,7 @@ Sk.builtin.int_ = Sk.abstr.buildNativeClass("int", { if (Sk.misceval.isTrue(signed)) { /** @todo - to_bytes with signed=True */ throw new Sk.builtin.NotImplementedError( - "from_bytes with signed=True is not yet implemented in Skulpt" + "to_bytes with signed=True is not yet implemented in Skulpt" ); } if (this.nb$isnegative()) { @@ -334,7 +342,7 @@ Sk.builtin.int_ = Sk.abstr.buildNativeClass("int", { j += 2; } - if (byteorder === s_lit) { + if (littleEndian) { u8.reverse(); } return new Sk.builtin.bytes(u8); @@ -989,4 +997,7 @@ const INTERNED_INT = []; for (let i = -5; i < 257; i++) { INTERNED_INT[i] = Object.create(Sk.builtin.int_.prototype, {v: {value: i}}); } -const INT_ZERO = INTERNED_INT[0]; \ No newline at end of file +const INT_ZERO = INTERNED_INT[0]; +// from_bytes and to_bytes +const S_LITTLE = new Sk.builtin.str("little"); +const S_BIG = new Sk.builtin.str("big"); \ No newline at end of file From a30dd4bb1392108d412ffb1680a11017b388e7e6 Mon Sep 17 00:00:00 2001 From: stu Date: Thu, 10 Nov 2022 22:11:44 +0800 Subject: [PATCH 071/137] int.from_bytes, to_bytes - extract the little/big function --- src/int.js | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/src/int.js b/src/int.js index 2baf45477a..cae043c23c 100644 --- a/src/int.js +++ b/src/int.js @@ -230,15 +230,7 @@ Sk.builtin.int_ = Sk.abstr.buildNativeClass("int", { kws, [Sk.builtin.bool.false$] ); - let littleEndian; - if (S_LITTLE.ob$eq(byteorder)) { - littleEndian = 1; - } else if (S_BIG.ob$eq(byteorder)) { - littleEndian = 0; - } else { - throw new Sk.builtin.ValueError("byteorder must be either 'little' or 'big'"); - } - + const littleEndian = isLittleEndian(byteorder); if (!(bytes instanceof Sk.builtin.bytes)) { // not quite right - we should call pyObjectBytes - which fails on integers // but good enough for now @@ -298,15 +290,7 @@ Sk.builtin.int_ = Sk.abstr.buildNativeClass("int", { kws, [Sk.builtin.bool.false$] ); - let littleEndian; - if (S_LITTLE.ob$eq(byteorder)) { - littleEndian = 1; - } else if (S_BIG.ob$eq(byteorder)) { - littleEndian = 0; - } else { - throw new Sk.builtin.ValueError("byteorder must be either 'little' or 'big'"); - } - + const littleEndian = isLittleEndian(byteorder); length = Sk.misceval.asIndexSized(length, Sk.builtin.OverflowError); if (length < 0) { throw new Sk.builtin.ValueError("length argument must be non-negative"); @@ -998,6 +982,18 @@ for (let i = -5; i < 257; i++) { INTERNED_INT[i] = Object.create(Sk.builtin.int_.prototype, {v: {value: i}}); } const INT_ZERO = INTERNED_INT[0]; + // from_bytes and to_bytes -const S_LITTLE = new Sk.builtin.str("little"); -const S_BIG = new Sk.builtin.str("big"); \ No newline at end of file +function isLittleEndian(byteorder) { + if (!Sk.builtin.checkString(byteorder)) { + throw new Sk.builtin.TypeError("'byteorder' must be str, not " + Sk.abstr.typeName(byteorder)); + } + byteorder = byteorder.toString(); + if (byteorder === "little") { + return 1; + } else if (byteorder === "big") { + return 0; + } else { + throw new Sk.builtin.ValueError("byteorder must be either 'little' or 'big'"); + } +} \ No newline at end of file From 7868428a5a63ac22851239d2d59e15a23f94350b Mon Sep 17 00:00:00 2001 From: Jeff Plumb Date: Fri, 15 Oct 2021 20:47:37 +1100 Subject: [PATCH 072/137] Change breakpoint in while statement for debugging Currently the breakpoint in a while loop is set to be after the test. This means that when implementing a line by line debugger, the final test that causes the while loop to terminate is not shown. By changing the position of the debug block, the final test will be shown. --- src/compile.js | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/compile.js b/src/compile.js index bb8d63bb7d..84c966e8df 100644 --- a/src/compile.js +++ b/src/compile.js @@ -1392,19 +1392,6 @@ Compiler.prototype.cwhile = function (s) { this._jump(top); this.setBlock(top); - next = this.newBlock("after while"); - orelse = s.orelse.length > 0 ? this.newBlock("while orelse") : null; - body = this.newBlock("while body"); - - this.annotateSource(s); - this._jumpfalse(this.vexpr(s.test), orelse ? orelse : next); - this._jump(body); - - this.pushBreakBlock(next); - this.pushContinueBlock(top); - - this.setBlock(body); - if ((Sk.debugging || Sk.killableWhile) && this.u.canSuspend) { var suspType = "Sk.delay"; var debugBlock = this.newBlock("debug breakpoint for line "+s.lineno); @@ -1419,6 +1406,19 @@ Compiler.prototype.cwhile = function (s) { this.u.doesSuspend = true; } + next = this.newBlock("after while"); + orelse = s.orelse.length > 0 ? this.newBlock("while orelse") : null; + body = this.newBlock("while body"); + + this.annotateSource(s); + this._jumpfalse(this.vexpr(s.test), orelse ? orelse : next); + this._jump(body); + + this.pushBreakBlock(next); + this.pushContinueBlock(top); + + this.setBlock(body); + this.vseqstmt(s.body); this._jump(top); From 5ab2b9cbbb5a9aef60fb15fb4bed06d1a619a580 Mon Sep 17 00:00:00 2001 From: stu Date: Fri, 15 Jul 2022 17:18:52 +0800 Subject: [PATCH 073/137] fix bad variable declaration in compile code. --- src/compile.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/compile.js b/src/compile.js index 84c966e8df..cf392d1053 100644 --- a/src/compile.js +++ b/src/compile.js @@ -2074,10 +2074,14 @@ Compiler.prototype.buildcodeobj = function (n, coname, decorator_list, args, cal this.u.varDeclsCode += "\nvar $args = this.$resolveArgs($posargs,$kwargs)\n"; } const sup_i = kwarg ? 1 : 0; + let supDeclare = ""; for (let i = 0; i < funcArgs.length; i++) { - const sup = i === sup_i ? "$sup = " : ""; - this.u.varDeclsCode += "," + sup + funcArgs[i] + "=$args[" + i + "]"; + if (i === sup_i) { + supDeclare = `,$sup=${funcArgs[i]}`; + } + this.u.varDeclsCode += "," + funcArgs[i] + "=$args[" + i + "]"; } + this.u.varDeclsCode += supDeclare; this.u.varDeclsCode += ";\n"; } From 469dff8d9e2b37db1645c9da98c760f78204506c Mon Sep 17 00:00:00 2001 From: stu Date: Fri, 15 Jul 2022 22:02:41 +0800 Subject: [PATCH 074/137] simplify the code for setting the instance --- src/compile.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/compile.js b/src/compile.js index cf392d1053..4ec5b1d744 100644 --- a/src/compile.js +++ b/src/compile.js @@ -2073,15 +2073,13 @@ Compiler.prototype.buildcodeobj = function (n, coname, decorator_list, args, cal } else { this.u.varDeclsCode += "\nvar $args = this.$resolveArgs($posargs,$kwargs)\n"; } - const sup_i = kwarg ? 1 : 0; - let supDeclare = ""; for (let i = 0; i < funcArgs.length; i++) { - if (i === sup_i) { - supDeclare = `,$sup=${funcArgs[i]}`; - } this.u.varDeclsCode += "," + funcArgs[i] + "=$args[" + i + "]"; } - this.u.varDeclsCode += supDeclare; + const instanceForSuper = funcArgs[kwarg ? 1 : 0]; + if (instanceForSuper) { + this.u.varDeclsCode += `,$sup=${instanceForSuper}`; + } this.u.varDeclsCode += ";\n"; } From 3e03e309d42b1cdac5d8d312e84e48b1fe9ccca8 Mon Sep 17 00:00:00 2001 From: stu Date: Fri, 2 Sep 2022 18:38:45 +0800 Subject: [PATCH 075/137] func.__defaults__ should have a set method --- src/function.js | 20 +- test/unit3/test_funcattrs.py | 398 +++++++++++++++++++++++++++++++++++ 2 files changed, 412 insertions(+), 6 deletions(-) create mode 100644 test/unit3/test_funcattrs.py diff --git a/src/function.js b/src/function.js index 55e4cbf817..e9f485c178 100644 --- a/src/function.js +++ b/src/function.js @@ -142,8 +142,17 @@ Sk.builtin.func = Sk.abstr.buildNativeClass("function", { }, __defaults__: { $get() { - return new Sk.builtin.tuple(this.$defaults); - }, // technically this is a writable property but we'll leave it as read-only for now + return this.$defaults == null ? Sk.builtin.none.none$ : new Sk.builtin.tuple(this.$defaults); + }, + $set(v) { + if (v === undefined || Sk.builtin.checkNone(v)) { + this.$defaults = null; + } else if (!(v instanceof Sk.builtin.tuple)) { + throw new Sk.builtin.TypeError("__defaults__ must be set to a tuple object"); + } else { + this.$defaults = v.valueOf(); + } + } }, __doc__: { $get() { @@ -175,7 +184,7 @@ Sk.builtin.func = Sk.abstr.buildNativeClass("function", { this.co_kwonlyargcount = this.func_code.co_kwonlyargcount || 0; this.co_varargs = this.func_code.co_varargs; this.co_kwargs = this.func_code.co_kwargs; - this.$defaults = this.func_code.$defaults || []; + this.$defaults = this.func_code.$defaults; this.$kwdefs = this.func_code.$kwdefs || []; }, $resolveArgs, @@ -202,9 +211,8 @@ function $resolveArgs(posargs, kw) { if (co_kwonlyargcount === 0 && !this.co_kwargs && (!kw || kw.length === 0) && !this.co_varargs) { if (posargs.length == co_argcount) { return posargs; - } else if(posargs.length === 0 && this.$defaults && - this.$defaults.length === co_argcount) { - for (let i=0; i!=this.$defaults.length; i++) { + } else if (posargs.length === 0 && this.$defaults && this.$defaults.length === co_argcount) { + for (let i = 0; i != this.$defaults.length; i++) { posargs[i] = this.$defaults[i]; } return posargs; diff --git a/test/unit3/test_funcattrs.py b/test/unit3/test_funcattrs.py new file mode 100644 index 0000000000..8b8aae7011 --- /dev/null +++ b/test/unit3/test_funcattrs.py @@ -0,0 +1,398 @@ +import types +import unittest + + +def global_function(): + def inner_function(): + class LocalClass: + pass + global inner_global_function + def inner_global_function(): + def inner_function2(): + pass + return inner_function2 + return LocalClass + return lambda: inner_function + + +class FuncAttrsTest(unittest.TestCase): + def setUp(self): + class F: + def a(self): + pass + def b(): + return 3 + self.fi = F() + self.F = F + self.b = b + + def cannot_set_attr(self, obj, name, value, exceptions): + try: + setattr(obj, name, value) + except exceptions: + pass + else: + self.fail("shouldn't be able to set %s to %r" % (name, value)) + try: + delattr(obj, name) + except exceptions: + pass + else: + self.fail("shouldn't be able to del %s" % name) + + +class FunctionPropertiesTest(FuncAttrsTest): + # Include the external setUp method that is common to all tests + def test_module(self): + self.assertEqual(self.b.__module__, __name__) + + def test_dir_includes_correct_attrs(self): + self.b.known_attr = 7 + self.assertIn('known_attr', dir(self.b), + "set attributes not in dir listing of method") + # Test on underlying function object of method + self.F.a.known_attr = 7 + # TODO improve __dir__ implementation + # self.assertIn('known_attr', dir(self.fi.a), "set attribute on function " + # "implementations, should show up in next dir") + + def test_duplicate_function_equality(self): + # Body of `duplicate' is the exact same as self.b + def duplicate(): + 'my docstring' + return 3 + self.assertNotEqual(self.b, duplicate) + + # def test_copying___code__(self): + # def test(): pass + # self.assertEqual(test(), None) + # test.__code__ = self.b.__code__ + # self.assertEqual(test(), 3) # self.b always returns 3, arbitrarily + + # def test___globals__(self): + # self.assertIs(self.b.__globals__, globals()) + # self.cannot_set_attr(self.b, '__globals__', 2, + # (AttributeError, TypeError)) + + # def test___closure__(self): + # a = 12 + # def f(): print(a) + # c = f.__closure__ + # self.assertIsInstance(c, tuple) + # self.assertEqual(len(c), 1) + # # don't have a type object handy + # self.assertEqual(c[0].__class__.__name__, "cell") + # self.cannot_set_attr(f, "__closure__", c, AttributeError) + + # def test_empty_cell(self): + # def f(): print(a) + # try: + # f.__closure__[0].cell_contents + # except ValueError: + # pass + # else: + # self.fail("shouldn't be able to read an empty cell") + # a = 12 + + # def test_set_cell(self): + # a = 12 + # def f(): return a + # c = f.__closure__ + # c[0].cell_contents = 9 + # self.assertEqual(c[0].cell_contents, 9) + # self.assertEqual(f(), 9) + # self.assertEqual(a, 9) + # del c[0].cell_contents + # try: + # c[0].cell_contents + # except ValueError: + # pass + # else: + # self.fail("shouldn't be able to read an empty cell") + # with self.assertRaises(NameError): + # f() + # with self.assertRaises(UnboundLocalError): + # print(a) + + def test___name__(self): + self.assertEqual(self.b.__name__, 'b') + self.b.__name__ = 'c' + self.assertEqual(self.b.__name__, 'c') + self.b.__name__ = 'd' + self.assertEqual(self.b.__name__, 'd') + # __name__ and __name__ must be a string + self.cannot_set_attr(self.b, '__name__', 7, TypeError) + # __name__ must be available when in restricted mode. Exec will raise + # AttributeError if __name__ is not available on f. + s = """def f(): pass\nf.__name__""" + exec(s, {'__builtins__': {}}) + # Test on methods, too + self.assertEqual(self.fi.a.__name__, 'a') + self.cannot_set_attr(self.fi.a, "__name__", 'a', AttributeError) + + # def test___qualname__(self): + # # PEP 3155 + # self.assertEqual(self.b.__qualname__, 'FuncAttrsTest.setUp..b') + # self.assertEqual(FuncAttrsTest.setUp.__qualname__, 'FuncAttrsTest.setUp') + # self.assertEqual(global_function.__qualname__, 'global_function') + # self.assertEqual(global_function().__qualname__, + # 'global_function..') + # self.assertEqual(global_function()().__qualname__, + # 'global_function..inner_function') + # self.assertEqual(global_function()()().__qualname__, + # 'global_function..inner_function..LocalClass') + # self.assertEqual(inner_global_function.__qualname__, 'inner_global_function') + # self.assertEqual(inner_global_function().__qualname__, 'inner_global_function..inner_function2') + # self.b.__qualname__ = 'c' + # self.assertEqual(self.b.__qualname__, 'c') + # self.b.__qualname__ = 'd' + # self.assertEqual(self.b.__qualname__, 'd') + # # __qualname__ must be a string + # self.cannot_set_attr(self.b, '__qualname__', 7, TypeError) + + # def test___code__(self): + # num_one, num_two = 7, 8 + # def a(): pass + # def b(): return 12 + # def c(): return num_one + # def d(): return num_two + # def e(): return num_one, num_two + # for func in [a, b, c, d, e]: + # self.assertEqual(type(func.__code__), types.CodeType) + # self.assertEqual(c(), 7) + # self.assertEqual(d(), 8) + # d.__code__ = c.__code__ + # self.assertEqual(c.__code__, d.__code__) + # self.assertEqual(c(), 7) + # # self.assertEqual(d(), 7) + # try: + # b.__code__ = c.__code__ + # except ValueError: + # pass + # else: + # self.fail("__code__ with different numbers of free vars should " + # "not be possible") + # try: + # e.__code__ = d.__code__ + # except ValueError: + # pass + # else: + # self.fail("__code__ with different numbers of free vars should " + # "not be possible") + + def test_blank_func_defaults(self): + self.assertEqual(self.b.__defaults__, None) + del self.b.__defaults__ + self.assertEqual(self.b.__defaults__, None) + + def test_func_default_args(self): + def first_func(a, b): + return a+b + def second_func(a=1, b=2): + return a+b + self.assertEqual(first_func.__defaults__, None) + self.assertEqual(second_func.__defaults__, (1, 2)) + first_func.__defaults__ = (1, 2) + self.assertEqual(first_func.__defaults__, (1, 2)) + self.assertEqual(first_func(), 3) + self.assertEqual(first_func(3), 5) + self.assertEqual(first_func(3, 5), 8) + del second_func.__defaults__ + self.assertEqual(second_func.__defaults__, None) + try: + second_func() + except TypeError: + pass + else: + self.fail("__defaults__ does not update; deleting it does not " + "remove requirement") + + +class InstancemethodAttrTest(FuncAttrsTest): + + def test___class__(self): + self.assertEqual(self.fi.a.__self__.__class__, self.F) + self.cannot_set_attr(self.fi.a, "__class__", self.F, TypeError) + + def test___func__(self): + self.assertEqual(self.fi.a.__func__, self.F.a) + self.cannot_set_attr(self.fi.a, "__func__", self.F.a, AttributeError) + + def test___self__(self): + self.assertEqual(self.fi.a.__self__, self.fi) + self.cannot_set_attr(self.fi.a, "__self__", self.fi, AttributeError) + + def test___func___non_method(self): + # Behavior should be the same when a method is added via an attr + # assignment + self.fi.id = types.MethodType(id, self.fi) + self.assertEqual(self.fi.id(), id(self.fi)) + # Test usage + try: + self.fi.id.unknown_attr + except AttributeError: + pass + else: + self.fail("using unknown attributes should raise AttributeError") + # Test assignment and deletion + self.cannot_set_attr(self.fi.id, 'unknown_attr', 2, AttributeError) + + +class ArbitraryFunctionAttrTest(FuncAttrsTest): + def test_set_attr(self): + self.b.known_attr = 7 + self.assertEqual(self.b.known_attr, 7) + try: + self.fi.a.known_attr = 7 + except AttributeError: + pass + else: + self.fail("setting attributes on methods should raise error") + + def test_delete_unknown_attr(self): + try: + del self.b.unknown_attr + except AttributeError: + pass + else: + self.fail("deleting unknown attribute should raise TypeError") + + def test_unset_attr(self): + for func in [self.b, self.fi.a]: + try: + func.non_existent_attr + except AttributeError: + pass + else: + self.fail("using unknown attributes should raise " + "AttributeError") + + +class FunctionDictsTest(FuncAttrsTest): + def test_setting_dict_to_invalid(self): + self.cannot_set_attr(self.b, '__dict__', None, TypeError) + # from collections import UserDict + # d = UserDict({'known_attr': 7}) + # self.cannot_set_attr(self.fi.a.__func__, '__dict__', d, TypeError) + + def test_setting_dict_to_valid(self): + d = {'known_attr': 7} + self.b.__dict__ = d + # Test assignment + self.assertIs(d, self.b.__dict__) + # ... and on all the different ways of referencing the method's func + self.F.a.__dict__ = d + self.assertIs(d, self.fi.a.__func__.__dict__) + self.assertIs(d, self.fi.a.__dict__) + # Test value + self.assertEqual(self.b.known_attr, 7) + self.assertEqual(self.b.__dict__['known_attr'], 7) + # ... and again, on all the different method's names + self.assertEqual(self.fi.a.__func__.known_attr, 7) + self.assertEqual(self.fi.a.known_attr, 7) + + def test_delete___dict__(self): + try: + del self.b.__dict__ + except TypeError: + pass + else: + self.fail("deleting function dictionary should raise TypeError") + + def test_unassigned_dict(self): + self.assertEqual(self.b.__dict__, {}) + + def test_func_as_dict_key(self): + value = "Some string" + d = {} + d[self.b] = value + self.assertEqual(d[self.b], value) + + +class FunctionDocstringTest(FuncAttrsTest): + def test_set_docstring_attr(self): + self.assertEqual(self.b.__doc__, None) + docstr = "A test method that does nothing" + self.b.__doc__ = docstr + self.F.a.__doc__ = docstr + self.assertEqual(self.b.__doc__, docstr) + self.assertEqual(self.fi.a.__doc__, docstr) + self.cannot_set_attr(self.fi.a, "__doc__", docstr, AttributeError) + + def test_delete_docstring(self): + self.b.__doc__ = "The docstring" + del self.b.__doc__ + self.assertEqual(self.b.__doc__, None) + + +def cell(value): + """Create a cell containing the given value.""" + def f(): + print(a) + a = value + return f.__closure__[0] + +def empty_cell(empty=True): + """Create an empty cell.""" + def f(): + print(a) + # the intent of the following line is simply "if False:"; it's + # spelt this way to avoid the danger that a future optimization + # might simply remove an "if False:" code block. + if not empty: + a = 1729 + return f.__closure__[0] + + +# class CellTest(unittest.TestCase): +# def test_comparison(self): +# # These tests are here simply to exercise the comparison code; +# # their presence should not be interpreted as providing any +# # guarantees about the semantics (or even existence) of cell +# # comparisons in future versions of CPython. +# self.assertTrue(cell(2) < cell(3)) +# self.assertTrue(empty_cell() < cell('saturday')) +# self.assertTrue(empty_cell() == empty_cell()) +# self.assertTrue(cell(-36) == cell(-36.0)) +# self.assertTrue(cell(True) > empty_cell()) + + +class StaticMethodAttrsTest(unittest.TestCase): + def test_func_attribute(self): + def f(): + pass + + c = classmethod(f) + self.assertTrue(c.__func__ is f) + + s = staticmethod(f) + self.assertTrue(s.__func__ is f) + + +# class BuiltinFunctionPropertiesTest(unittest.TestCase): +# # XXX Not sure where this should really go since I can't find a +# # test module specifically for builtin_function_or_method. + +# def test_builtin__qualname__(self): +# import time + +# # builtin function: +# self.assertEqual(len.__qualname__, 'len') +# self.assertEqual(time.time.__qualname__, 'time') + +# # builtin classmethod: +# self.assertEqual(dict.fromkeys.__qualname__, 'dict.fromkeys') +# self.assertEqual(float.__getformat__.__qualname__, +# 'float.__getformat__') + +# # builtin staticmethod: +# self.assertEqual(str.maketrans.__qualname__, 'str.maketrans') +# self.assertEqual(bytes.maketrans.__qualname__, 'bytes.maketrans') + +# # builtin bound instance method: +# self.assertEqual([1, 2, 3].append.__qualname__, 'list.append') +# self.assertEqual({'foo': 'bar'}.pop.__qualname__, 'dict.pop') + + +if __name__ == "__main__": + unittest.main() From f569bb67998c6c7915ff5433108e93876188b4c8 Mon Sep 17 00:00:00 2001 From: stu Date: Tue, 19 Apr 2022 23:05:07 +0800 Subject: [PATCH 076/137] fix reserved words don't need to be mangled in annotations - close #1428 --- src/compile.js | 2 +- test/unit3/test_skulpt_bugs.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/compile.js b/src/compile.js index 4ec5b1d744..451593513f 100644 --- a/src/compile.js +++ b/src/compile.js @@ -1101,7 +1101,7 @@ Compiler.prototype.cannassign = function (s) { if (s.simple && (this.u.ste.blockType === Sk.SYMTAB_CONSTS.ClassBlock || this.u.ste.blockType == Sk.SYMTAB_CONSTS.ModuleBlock)) { this.u.hasAnnotations = true; const val = this.vexpr(s.annotation); - let mangled = fixReserved(mangleName(this.u.private_, target.id).v); + let mangled = mangleName(this.u.private_, target.id).v; const key = this.makeConstant("new Sk.builtin.str('" + mangled + "')"); this.chandlesubscr(Sk.astnodes.Store, "$loc.__annotations__", key, val); } diff --git a/test/unit3/test_skulpt_bugs.py b/test/unit3/test_skulpt_bugs.py index 15136f6a4b..ddac42494c 100644 --- a/test/unit3/test_skulpt_bugs.py +++ b/test/unit3/test_skulpt_bugs.py @@ -41,5 +41,14 @@ def f(x): +class An: + name: str + __foo: int + +class TestAnnotations(unittest.TestCase): + def test_bug_1428(self): + annotations = An.__annotations__ + self.assertEqual(annotations, {"name": str, "_An__foo": int}) + if __name__ == "__main__": unittest.main() From 28350efff5ef300c426ebb4c9ecbb48220b6ceda Mon Sep 17 00:00:00 2001 From: stu Date: Sat, 5 Feb 2022 11:39:21 +0800 Subject: [PATCH 077/137] dir: make make it match cpython --- src/abstract.js | 10 +++++++++ src/constants.js | 1 + src/object.js | 40 +++++++++++++++++++++------------- src/type.js | 32 +++------------------------- test/unit3/test_descr.py | 46 ++++++++++++++++++---------------------- 5 files changed, 60 insertions(+), 69 deletions(-) diff --git a/src/abstract.js b/src/abstract.js index 77fd09bb20..2f28d63391 100644 --- a/src/abstract.js +++ b/src/abstract.js @@ -921,6 +921,16 @@ Sk.abstr.lookupSpecial = function (obj, pyName) { }; Sk.exportSymbol("Sk.abstr.lookupSpecial", Sk.abstr.lookupSpecial); +Sk.abstr.lookupAttr = function (obj, pyName) { + try { + return obj.tp$getattr(pyName); + } catch (e) { + if (!(e instanceof Sk.builtin.AttributeError)) { + throw e; + } + } +}; + Sk.abstr.typeLookup = function (type_obj, pyName) { const res = type_obj.$typeLookup(pyName); diff --git a/src/constants.js b/src/constants.js index 2effb22104..3679e412ed 100644 --- a/src/constants.js +++ b/src/constants.js @@ -13,6 +13,7 @@ Sk.builtin.str.$imag = new Sk.builtin.str("imag"); Sk.builtin.str.$real = new Sk.builtin.str("real"); Sk.builtin.str.$abs = new Sk.builtin.str("__abs__"); +Sk.builtin.str.$bases = new Sk.builtin.str("__bases__"); Sk.builtin.str.$bytes = new Sk.builtin.str("__bytes__"); Sk.builtin.str.$call = new Sk.builtin.str("__call__"); Sk.builtin.str.$class = new Sk.builtin.str("__class__"); diff --git a/src/object.js b/src/object.js index fba55290e3..839916d61a 100644 --- a/src/object.js +++ b/src/object.js @@ -108,22 +108,19 @@ Sk.builtin.object = Sk.abstr.buildNativeClass("object", { methods: { __dir__: { $meth: function __dir__() { - let dir = []; - if (this.$d) { - if (this.$d instanceof Sk.builtin.dict) { - dir = this.$d.sk$asarray(); - } else { - for (let key in this.$d) { - dir.push(new Sk.builtin.str(key)); - } - } + let dict = Sk.abstr.lookupAttr(this, Sk.builtin.str.$dict); + if (dict === undefined) { + dict = new Sk.builtin.dict([]); + } else if (!(dict instanceof Sk.builtin.dict)) { + dict = new Sk.builtin.dict([]); + } else { + dict = dict.dict$copy(); + } + const cls = Sk.abstr.lookupAttr(this, Sk.builtin.str.$class); + if (cls !== undefined) { + cls.$mergeClassDict(dict); } - // here we use the type.__dir__ implementation - const type_dir = Sk.misceval.callsimArray(Sk.builtin.type.prototype.__dir__, [this.ob$type]); - // put the dict keys before the prototype keys - dir.push(...type_dir.v); - type_dir.v = dir; - return type_dir; + return new Sk.builtin.list(dict.sk$asarray()); }, $flags: { NoArgs: true }, $doc: "Default dir() implementation.", @@ -160,6 +157,19 @@ Sk.builtin.object = Sk.abstr.buildNativeClass("object", { sk$attrError() { return "'" + this.tp$name + "' object"; }, + $mergeClassDict(dict) { + const classDict = Sk.abstr.lookupAttr(this, Sk.builtin.str.$dict); + dict.dict$merge(classDict); + const bases = Sk.abstr.lookupAttr(this, Sk.builtin.str.$bases); + if (bases === undefined) { + return; + } + const n = Sk.builtin.len(bases).valueOf(); + for (let i = 0; i < n; i++) { + const base = Sk.abstr.objectGetItem(bases, new Sk.builtin.int_(i)); + base.$mergeClassDict(dict); + } + }, }, }); diff --git a/src/type.js b/src/type.js index f7ed4e3cfb..74ef6bbb69 100644 --- a/src/type.js +++ b/src/type.js @@ -754,35 +754,9 @@ Sk.builtin.type.prototype.tp$methods = /**@lends {Sk.builtin.type.prototype}*/ { }, __dir__: { $meth: function __dir__() { - const seen = new Set(); - const dir = []; - function push_or_continue(attr) { - if (attr in Sk.reservedWords_) { - return; - } - attr = Sk.unfixReserved(attr); - if (attr.indexOf("$") !== -1) { - return; - } - if (!seen.has(attr)) { - seen.add(attr); - dir.push(new Sk.builtin.str(attr)); - } - } - if (this.prototype.sk$prototypical) { - for (let attr in this.prototype) { - push_or_continue(attr); - } - } else { - const mro = this.prototype.tp$mro; - for (let i = 0; i < mro.length; i++) { - const attrs = Object.getOwnPropertyNames(mro[i].prototype); - for (let j = 0; j < attrs.length; j++) { - push_or_continue(attrs[j]); - } - } - } - return new Sk.builtin.list(dir); + const dict = new Sk.builtin.dict([]); + this.$mergeClassDict(dict); + return new Sk.builtin.list(dict.sk$asarray()); }, $flags: { NoArgs: true }, $doc: "Specialized __dir__ implementation for types.", diff --git a/test/unit3/test_descr.py b/test/unit3/test_descr.py index 0513e6633d..1c7c1c3595 100644 --- a/test/unit3/test_descr.py +++ b/test/unit3/test_descr.py @@ -2493,10 +2493,8 @@ def test_dir(self): # del junk # Just make sure these don't blow up! - for arg in 2, 2, 2j, 2e0, [2], "2", (2,), {2:2}, type, self.test_dir: + for arg in 2, 2, 2j, 2e0, [2], "2", b"2", (2,), {2:2}, type, self.test_dir: dir(arg) - # for arg in 2, 2, 2j, 2e0, [2], "2", b"2", (2,), {2:2}, type, self.test_dir: - # dir(arg) # Test dir on new-style classes. Since these have object as a # base class, a lot more gets sucked in. @@ -2553,34 +2551,32 @@ def getdict(self): m2instance.b = 2 m2instance.a = 1 self.assertEqual(m2instance.__dict__, "Not a dict!") - try: + with self.assertRaises(TypeError): dir(m2instance) - except TypeError: - pass # Two essentially featureless objects, (Ellipsis just inherits stuff # from object. - # self.assertEqual(dir(object()), dir(Ellipsis)) + self.assertEqual(dir(object()), dir(Ellipsis)) # Nasty test case for proxied objects - # class Wrapper(object): - # def __init__(self, obj): - # self.__obj = obj - # def __repr__(self): - # return "Wrapper(%s)" % repr(self.__obj) - # def __getitem__(self, key): - # return Wrapper(self.__obj[key]) - # def __len__(self): - # return len(self.__obj) - # def __getattr__(self, name): - # return Wrapper(getattr(self.__obj, name)) - - # class C(object): - # def __getclass(self): - # return Wrapper(type(self)) - # __class__ = property(__getclass) - - # dir(C()) # This used to segfault + class Wrapper(object): + def __init__(self, obj): + self.__obj = obj + def __repr__(self): + return "Wrapper(%s)" % repr(self.__obj) + def __getitem__(self, key): + return Wrapper(self.__obj[key]) + def __len__(self): + return len(self.__obj) + def __getattr__(self, name): + return Wrapper(getattr(self.__obj, name)) + + class C(object): + def __getclass(self): + return Wrapper(type(self)) + __class__ = property(__getclass) + + dir(C()) # This used to segfault def test_supers(self): # Testing super... From 69fc3f14fff842da673c617084a4846fcb9d3d4a Mon Sep 17 00:00:00 2001 From: stu Date: Tue, 8 Mar 2022 21:18:29 +0800 Subject: [PATCH 078/137] Fix float and int don't support __index__ --- src/float.js | 4 +- src/int.js | 5 +- src/misceval.js | 4 +- src/slotdefs.js | 2 + test/unit3/test_class.py | 11 +- test/unit3/test_float.py | 77 +++++++ test/unit3/test_index.py | 480 +++++++++++++++++---------------------- test/unit3/test_int.py | 68 +++++- 8 files changed, 361 insertions(+), 290 deletions(-) diff --git a/src/float.js b/src/float.js index 77e9aca439..13ffe45bca 100644 --- a/src/float.js +++ b/src/float.js @@ -61,8 +61,10 @@ Sk.builtin.float_ = Sk.abstr.buildNativeClass("float", { // is args always an empty list? if (arg === undefined) { x = new Sk.builtin.float_(0.0); - } else if (arg.nb$float) { + } else if (arg.nb$float !== undefined) { x = arg.nb$float(); + } else if (arg.nb$index !== undefined) { + x = new Sk.builtin.int_(arg.nb$index()).nb$float(); } else if (Sk.builtin.checkString(arg)) { x = _str_to_float(arg.v); } diff --git a/src/int.js b/src/int.js index cae043c23c..1a242fdc1a 100644 --- a/src/int.js +++ b/src/int.js @@ -421,6 +421,7 @@ Sk.builtin.int_ = Sk.abstr.buildNativeClass("int", { valueOf() { return this.v; }, + sk$int: true, }, }); @@ -872,8 +873,10 @@ function getInt(x, base) { return new Sk.builtin.int_(Sk.str2number(x.v, base)); } else if (base !== null) { throw new Sk.builtin.TypeError("int() can't convert non-string with explicit base"); - } else if (x.nb$int) { + } else if (x.nb$int !== undefined) { return x.nb$int(); + } else if (x.nb$index !== undefined) { + return new Sk.builtin.int_(x.nb$index()); } if ((func = Sk.abstr.lookupSpecial(x, Sk.builtin.str.$trunc))) { diff --git a/src/misceval.js b/src/misceval.js index 2ee2370b25..c0895cd4ed 100644 --- a/src/misceval.js +++ b/src/misceval.js @@ -92,7 +92,9 @@ Sk.exportSymbol("Sk.misceval.isIndex", Sk.misceval.isIndex); function asIndex(index) { if (index === null || index === undefined) { return; - } else if (index.nb$index) { + } else if (index.sk$int === true) { + return index.v; + } else if (index.nb$index !== undefined) { return index.nb$index(); // this slot will check the return value is a number / JSBI.BigInt. } else if (typeof index === "number" && Number.isInteger(index)) { return index; diff --git a/src/slotdefs.js b/src/slotdefs.js index 5c81dd7f6e..1169a8eae1 100644 --- a/src/slotdefs.js +++ b/src/slotdefs.js @@ -1864,6 +1864,7 @@ Sk.subSlots = { nb$int: "__int__", nb$long: "__long__", nb$float: "__float__", + nb$index: "__index__", nb$add: "__add__", nb$reflected_add: "__radd__", nb$inplace_add: "__iadd__", @@ -2072,6 +2073,7 @@ Sk.dunderToSkulpt = { __pos__: "nb$positive", __int__: "nb$int", __float__: "nb$float", + __index__: "nb$index", __add__: "nb$add", __radd__: "nb$reflected_add", diff --git a/test/unit3/test_class.py b/test/unit3/test_class.py index 2d3a43673c..24e9c36b46 100644 --- a/test/unit3/test_class.py +++ b/test/unit3/test_class.py @@ -374,15 +374,8 @@ def __int__(self): def index(x): return [][x] - # self.assertRaises(TypeError, float, BadTypeClass()) - self.assertRaises(TypeError, complex, BadTypeClass()) - # self.assertRaises(TypeError, str, BadTypeClass()) - # self.assertRaises(TypeError, repr, BadTypeClass()) - self.assertRaises(TypeError, bin, BadTypeClass()) - self.assertRaises(TypeError, oct, BadTypeClass()) - self.assertRaises(TypeError, hex, BadTypeClass()) - # self.assertRaises(TypeError, bool, BadTypeClass()) - self.assertRaises(TypeError, index, BadTypeClass()) + for f in [float, complex, str, repr, bytes, bin, oct, hex, bool, index]: + self.assertRaises(TypeError, f, BadTypeClass()) def testHashStuff(self): diff --git a/test/unit3/test_float.py b/test/unit3/test_float.py index 8aa5c34c35..659f17fdde 100644 --- a/test/unit3/test_float.py +++ b/test/unit3/test_float.py @@ -1,6 +1,8 @@ +# TODO include the rest of the tests for this file import sys import unittest import math +import time from test_grammar import (VALID_UNDERSCORE_LITERALS, INVALID_UNDERSCORE_LITERALS) @@ -9,6 +11,81 @@ NAN = float("nan") + +class FloatSubclass(float): + pass + +class OtherFloatSubclass(float): + pass +class GeneralFloatCases(unittest.TestCase): + def test_floatconversion(self): + # Make sure that calls to __float__() work properly + class Foo1(object): + def __float__(self): + return 42. + + class Foo2(float): + def __float__(self): + return 42. + + class Foo3(float): + def __new__(cls, value=0.): + return float.__new__(cls, 2*value) + + def __float__(self): + return self + + class Foo4(float): + def __float__(self): + return 42 + + # Issue 5759: __float__ not called on str subclasses (though it is on + # unicode subclasses). + class FooStr(str): + def __float__(self): + return float(str(self)) + 1 + + self.assertEqual(float(Foo1()), 42.) + self.assertEqual(float(Foo2()), 42.) + # with self.assertWarns(DeprecationWarning): + # self.assertEqual(float(Foo3(21)), 42.) + self.assertRaises(TypeError, float, Foo4(42)) + self.assertEqual(float(FooStr('8')), 9.) + + class Foo5: + def __float__(self): + return "" + self.assertRaises(TypeError, time.sleep, Foo5()) + + # Issue #24731 + class F: + def __float__(self): + return OtherFloatSubclass(42.) + # with self.assertWarns(DeprecationWarning): + # self.assertEqual(float(F()), 42.) + # with self.assertWarns(DeprecationWarning): + # self.assertIs(type(float(F())), float) + # with self.assertWarns(DeprecationWarning): + # self.assertEqual(FloatSubclass(F()), 42.) + # with self.assertWarns(DeprecationWarning): + # self.assertIs(type(FloatSubclass(F())), FloatSubclass) + + class MyIndex: + def __init__(self, value): + self.value = value + def __index__(self): + return self.value + + self.assertEqual(float(MyIndex(42)), 42.0) + self.assertRaises(OverflowError, float, MyIndex(2**2000)) + + class MyInt: + def __int__(self): + return 42 + + self.assertRaises(TypeError, float, MyInt()) + + class FloatTestCases(unittest.TestCase): def test_conjugate(self): self.assertEqual(float(3.0).conjugate(), 3.0) diff --git a/test/unit3/test_index.py b/test/unit3/test_index.py index b62676371b..9c342ebfc6 100644 --- a/test/unit3/test_index.py +++ b/test/unit3/test_index.py @@ -1,186 +1,200 @@ import unittest import operator -#from sys import maxint -#maxsize = test_support.MAX_Py_ssize_t -#minsize = -maxsize-1 +import sys +maxsize = sys.maxsize -class oldstyle: +class newstyle: def __index__(self): return self.ind -# class newstyle(object): -# def __index__(self): -# return self.ind - class TrapInt(int): def __index__(self): - return self - -class TrapLong(int): - # changed to int - def __index__(self): - return self + return int(self) class BaseTestCase(unittest.TestCase): def setUp(self): - self.o = oldstyle() - #self.n = newstyle() + self.o = newstyle() + self.n = newstyle() def test_basic(self): self.o.ind = -2 - #self.n.ind = 2 + self.n.ind = 2 self.assertEqual(operator.index(self.o), -2) - #self.assertEqual(operator.index(self.n), 2) + self.assertEqual(operator.index(self.n), 2) def test_slice(self): self.o.ind = 1 - #self.n.ind = 2 + self.n.ind = 2 slc = slice(self.o, self.o, self.o) check_slc = slice(1, 1, 1) self.assertEqual(slc.indices(self.o), check_slc.indices(1)) - #slc = slice(self.n, self.n, self.n) - #check_slc = slice(2, 2, 2) - #self.assertEqual(slc.indices(self.n), check_slc.indices(2)) + slc = slice(self.n, self.n, self.n) + check_slc = slice(2, 2, 2) + self.assertEqual(slc.indices(self.n), check_slc.indices(2)) def test_wrappers(self): self.o.ind = 4 - #self.n.ind = 5 - #self.assertEqual(6.__index__(), 6) - #self.assertEqual(-7L.__index__(), -7) + self.n.ind = 5 + self.assertEqual(6 .__index__(), 6) + self.assertEqual(-7 .__index__(), -7) self.assertEqual(self.o.__index__(), 4) - #self.assertEqual(self.n.__index__(), 5) - #self.assertEqual(True.__index__(), 1) - #self.assertEqual(False.__index__(), 0) + self.assertEqual(self.n.__index__(), 5) + self.assertEqual(True.__index__(), 1) + self.assertEqual(False.__index__(), 0) def test_subclasses(self): - r = range(10) + r = list(range(10)) self.assertEqual(r[TrapInt(5):TrapInt(10)], r[5:10]) - self.assertEqual(r[TrapLong(5):TrapLong(10)], r[5:10]) self.assertEqual(slice(TrapInt()).indices(0), (0,0,1)) - self.assertEqual(slice(TrapLong(0)).indices(0), (0,0,1)) def test_error(self): self.o.ind = 'dumb' - #self.n.ind = 'bad' + self.n.ind = 'bad' self.assertRaises(TypeError, operator.index, self.o) - #self.assertRaises(TypeError, operator.index, self.n) + self.assertRaises(TypeError, operator.index, self.n) self.assertRaises(TypeError, slice(self.o).indices, 0) - #self.assertRaises(TypeError, slice(self.n).indices, 0) - - -# class SeqTestCase(unittest.TestCase): -# # This test case isn't run directly. It just defines common tests -# # to the different sequence types below -# def setUp(self): -# self.o = oldstyle() -# #self.n = newstyle() -# self.o2 = oldstyle() -# #self.n2 = newstyle() -# -# def test_index(self): -# self.o.ind = -2 -# #self.n.ind = 2 -# #self.assertEqual(self.seq[self.n], self.seq[2]) -# self.assertEqual(self.seq[self.o], self.seq[-2]) -# -# def test_slice(self): -# self.o.ind = 1 -# self.o2.ind = 3 -# #self.n.ind = 2 -# #self.n2.ind = 4 -# self.assertEqual(self.seq[self.o:self.o2], self.seq[1:3]) -# #self.assertEqual(self.seq[self.n:self.n2], self.seq[2:4]) -# -# def test_slice_bug7532a(self): -# seqlen = len(self.seq) -# self.o.ind = int(seqlen * 1.5) -# #self.n.ind = seqlen + 2 -# self.assertEqual(self.seq[self.o:], self.seq[0:0]) -# self.assertEqual(self.seq[:self.o], self.seq) -# #self.assertEqual(self.seq[self.n:], self.seq[0:0]) -# #self.assertEqual(self.seq[:self.n], self.seq) -# -# def test_slice_bug7532b(self): -# if isinstance(self.seq, ClassicSeq): -# self.skipTest('test fails for ClassicSeq') -# # These tests fail for ClassicSeq (see bug #7532) -# seqlen = len(self.seq) -# self.o2.ind = -seqlen - 2 -# #self.n2.ind = -int(seqlen * 1.5) -# self.assertEqual(self.seq[self.o2:], self.seq) -# self.assertEqual(self.seq[:self.o2], self.seq[0:0]) -# #self.assertEqual(self.seq[self.n2:], self.seq) -# #self.assertEqual(self.seq[:self.n2], self.seq[0:0]) -# -# def test_repeat(self): -# self.o.ind = 3 -# #self.n.ind = 2 -# self.assertEqual(self.seq * self.o, self.seq * 3) -# #self.assertEqual(self.seq * self.n, self.seq * 2) -# self.assertEqual(self.o * self.seq, self.seq * 3) -# #self.assertEqual(self.n * self.seq, self.seq * 2) -# -# def test_wrappers(self): -# self.o.ind = 4 -# #self.n.ind = 5 -# self.assertEqual(self.seq.__getitem__(self.o), self.seq[4]) -# self.assertEqual(self.seq.__mul__(self.o), self.seq * 4) -# self.assertEqual(self.seq.__rmul__(self.o), self.seq * 4) -# #self.assertEqual(self.seq.__getitem__(self.n), self.seq[5]) -# #self.assertEqual(self.seq.__mul__(self.n), self.seq * 5) -# #self.assertEqual(self.seq.__rmul__(self.n), self.seq * 5) -# -# def test_subclasses(self): -# self.assertEqual(self.seq[TrapInt()], self.seq[0]) -# self.assertEqual(self.seq[TrapLong()], self.seq[0]) -# -# def test_error(self): -# self.o.ind = 'dumb' -# #self.n.ind = 'bad' -# indexobj = lambda x, obj: obj.seq[x] -# self.assertRaises(TypeError, indexobj, self.o, self) -# #self.assertRaises(TypeError, indexobj, self.n, self) -# sliceobj = lambda x, obj: obj.seq[x:] -# self.assertRaises(TypeError, sliceobj, self.o, self) -# #self.assertRaises(TypeError, sliceobj, self.n, self) -# -# -# class ListTestCase(SeqTestCase): -# seq = [0,10,20,30,40,50] -# -# def test_setdelitem(self): -# self.o.ind = -2 -# #self.n.ind = 2 -# lst = list('ab!cdefghi!j') -# del lst[self.o] -# #del lst[self.n] -# lst[self.o] = 'X' -# #lst[self.n] = 'Y' -# #self.assertEqual(lst, list('abYdefghXj')) -# self.assertEqual(lst, list('ab!cdefghXj')) -# -# # lst = [5, 6, 7, 8, 9, 10, 11] -# # lst.__setitem__(self.n, "here") -# # self.assertEqual(lst, [5, 6, "here", 8, 9, 10, 11]) -# # lst.__delitem__(self.n) -# # self.assertEqual(lst, [5, 6, 8, 9, 10, 11]) -# -# def test_inplace_repeat(self): -# self.o.ind = 2 -# #self.n.ind = 3 -# lst = [6, 4] -# lst *= self.o -# self.assertEqual(lst, [6, 4, 6, 4]) -# #lst *= self.n -# #self.assertEqual(lst, [6, 4, 6, 4] * 3) -# -# #lst = [5, 6, 7, 8, 9, 11] -# #l2 = lst.__imul__(self.n) -# #self.assertIs(l2, lst) -# #self.assertEqual(lst, [5, 6, 7, 8, 9, 11] * 3) - - -class _BaseSeq: + self.assertRaises(TypeError, slice(self.n).indices, 0) + + def test_int_subclass_with_index(self): + # __index__ should be used when computing indices, even for int + # subclasses. See issue #17576. + class MyInt(int): + def __index__(self): + return int(str(self)) + 1 + + my_int = MyInt(7) + direct_index = my_int.__index__() + operator_index = operator.index(my_int) + self.assertEqual(direct_index, 8) + self.assertEqual(operator_index, 7) + # Both results should be of exact type int. + self.assertIs(type(direct_index), int) + #self.assertIs(type(operator_index), int) + + def test_index_returns_int_subclass(self): + class BadInt: + def __index__(self): + return True + + class BadInt2(int): + def __index__(self): + return True + + bad_int = BadInt() + # with self.assertWarns(DeprecationWarning): + n = operator.index(bad_int) + self.assertEqual(n, 1) + + bad_int = BadInt2() + n = operator.index(bad_int) + self.assertEqual(n, 0) + + +class SeqTestCase: + # This test case isn't run directly. It just defines common tests + # to the different sequence types below + def setUp(self): + self.o = newstyle() + self.n = newstyle() + self.o2 = newstyle() + self.n2 = newstyle() + + def test_index(self): + self.o.ind = -2 + self.n.ind = 2 + self.assertEqual(self.seq[self.n], self.seq[2]) + self.assertEqual(self.seq[self.o], self.seq[-2]) + + def test_slice(self): + self.o.ind = 1 + self.o2.ind = 3 + self.n.ind = 2 + self.n2.ind = 4 + self.assertEqual(self.seq[self.o:self.o2], self.seq[1:3]) + self.assertEqual(self.seq[self.n:self.n2], self.seq[2:4]) + + def test_slice_bug7532(self): + seqlen = len(self.seq) + self.o.ind = int(seqlen * 1.5) + self.n.ind = seqlen + 2 + self.assertEqual(self.seq[self.o:], self.seq[0:0]) + self.assertEqual(self.seq[:self.o], self.seq) + self.assertEqual(self.seq[self.n:], self.seq[0:0]) + self.assertEqual(self.seq[:self.n], self.seq) + self.o2.ind = -seqlen - 2 + self.n2.ind = -int(seqlen * 1.5) + self.assertEqual(self.seq[self.o2:], self.seq) + self.assertEqual(self.seq[:self.o2], self.seq[0:0]) + self.assertEqual(self.seq[self.n2:], self.seq) + self.assertEqual(self.seq[:self.n2], self.seq[0:0]) + + def test_repeat(self): + self.o.ind = 3 + self.n.ind = 2 + self.assertEqual(self.seq * self.o, self.seq * 3) + self.assertEqual(self.seq * self.n, self.seq * 2) + self.assertEqual(self.o * self.seq, self.seq * 3) + self.assertEqual(self.n * self.seq, self.seq * 2) + + def test_wrappers(self): + self.o.ind = 4 + self.n.ind = 5 + self.assertEqual(self.seq.__getitem__(self.o), self.seq[4]) + self.assertEqual(self.seq.__mul__(self.o), self.seq * 4) + self.assertEqual(self.seq.__rmul__(self.o), self.seq * 4) + self.assertEqual(self.seq.__getitem__(self.n), self.seq[5]) + self.assertEqual(self.seq.__mul__(self.n), self.seq * 5) + self.assertEqual(self.seq.__rmul__(self.n), self.seq * 5) + + def test_subclasses(self): + self.assertEqual(self.seq[TrapInt()], self.seq[0]) + + def test_error(self): + self.o.ind = 'dumb' + self.n.ind = 'bad' + indexobj = lambda x, obj: obj.seq[x] + self.assertRaises(TypeError, indexobj, self.o, self) + self.assertRaises(TypeError, indexobj, self.n, self) + sliceobj = lambda x, obj: obj.seq[x:] + self.assertRaises(TypeError, sliceobj, self.o, self) + self.assertRaises(TypeError, sliceobj, self.n, self) + + +class ListTestCase(SeqTestCase, unittest.TestCase): + seq = [0,10,20,30,40,50] + + def test_setdelitem(self): + self.o.ind = -2 + self.n.ind = 2 + lst = list('ab!cdefghi!j') + del lst[self.o] + del lst[self.n] + lst[self.o] = 'X' + lst[self.n] = 'Y' + self.assertEqual(lst, list('abYdefghXj')) + + lst = [5, 6, 7, 8, 9, 10, 11] + lst.__setitem__(self.n, "here") + self.assertEqual(lst, [5, 6, "here", 8, 9, 10, 11]) + lst.__delitem__(self.n) + self.assertEqual(lst, [5, 6, 8, 9, 10, 11]) + + def test_inplace_repeat(self): + self.o.ind = 2 + self.n.ind = 3 + lst = [6, 4] + lst *= self.o + self.assertEqual(lst, [6, 4, 6, 4]) + lst *= self.n + self.assertEqual(lst, [6, 4, 6, 4] * 3) + + lst = [5, 6, 7, 8, 9, 11] + l2 = lst.__imul__(self.n) + self.assertIs(l2, lst) + self.assertEqual(lst, [5, 6, 7, 8, 9, 11] * 3) + + +class NewSeq: def __init__(self, iterable): self._list = list(iterable) @@ -202,125 +216,59 @@ def __getitem__(self, index): return self._list[index] -class _GetSliceMixin: +class TupleTestCase(SeqTestCase, unittest.TestCase): + seq = (0,10,20,30,40,50) + +# class ByteArrayTestCase(SeqTestCase, unittest.TestCase): +# seq = bytearray(b"this is a test") + +class BytesTestCase(SeqTestCase, unittest.TestCase): + seq = b"this is a test" + +class StringTestCase(SeqTestCase, unittest.TestCase): + seq = "this is a test" - def __getslice__(self, i, j): - return self._list.__getslice__(i, j) +class NewSeqTestCase(SeqTestCase, unittest.TestCase): + seq = NewSeq((0,10,20,30,40,50)) - -class ClassicSeq(_BaseSeq): pass -#class NewSeq(_BaseSeq, object): pass -#class ClassicSeqDeprecated(_GetSliceMixin, ClassicSeq): pass -#class NewSeqDeprecated(_GetSliceMixin, NewSeq): pass - - -# class TupleTestCase(SeqTestCase): -# seq = (0,10,20,30,40,50) -# -# class StringTestCase(SeqTestCase): -# seq = "this is a test" - -# class ByteArrayTestCase(SeqTestCase): -# seq = bytearray("this is a test") - -# class UnicodeTestCase(SeqTestCase): -# seq = u"this is a test" - -# class ClassicSeqTestCase(SeqTestCase): -# seq = ClassicSeq((0,10,20,30,40,50)) - -# class NewSeqTestCase(SeqTestCase): -# seq = NewSeq((0,10,20,30,40,50)) - -# class ClassicSeqDeprecatedTestCase(SeqTestCase): -# seq = ClassicSeqDeprecated((0,10,20,30,40,50)) - -# class NewSeqDeprecatedTestCase(SeqTestCase): -# seq = NewSeqDeprecated((0,10,20,30,40,50)) - - -# class XRangeTestCase(unittest.TestCase): - -# def test_xrange(self): -# n = newstyle() -# n.ind = 5 -# self.assertEqual(xrange(1, 20)[n], 6) -# self.assertEqual(xrange(1, 20).__getitem__(n), 6) - -# class OverflowTestCase(unittest.TestCase): - -# def setUp(self): -# self.pos = 2**100 -# self.neg = -self.pos - -# def test_large_longs(self): -# self.assertEqual(self.pos.__index__(), self.pos) -# self.assertEqual(self.neg.__index__(), self.neg) - -# def _getitem_helper(self, base): -# class GetItem(base): -# def __len__(self): -# return maxint # cannot return long here -# def __getitem__(self, key): -# return key -# x = GetItem() -# self.assertEqual(x[self.pos], self.pos) -# self.assertEqual(x[self.neg], self.neg) -# self.assertEqual(x[self.neg:self.pos].indices(maxsize), -# (0, maxsize, 1)) -# self.assertEqual(x[self.neg:self.pos:1].indices(maxsize), -# (0, maxsize, 1)) - -# def _getslice_helper_deprecated(self, base): -# class GetItem(base): -# def __len__(self): -# return maxint # cannot return long here -# def __getitem__(self, key): -# return key -# def __getslice__(self, i, j): -# return i, j -# x = GetItem() -# self.assertEqual(x[self.pos], self.pos) -# self.assertEqual(x[self.neg], self.neg) -# self.assertEqual(x[self.neg:self.pos], (maxint+minsize, maxsize)) -# self.assertEqual(x[self.neg:self.pos:1].indices(maxsize), -# (0, maxsize, 1)) - -# def test_getitem(self): -# self._getitem_helper(object) -# with test_support.check_py3k_warnings(): -# self._getslice_helper_deprecated(object) - -# def test_getitem_classic(self): -# class Empty: pass -# # XXX This test fails (see bug #7532) -# #self._getitem_helper(Empty) -# with test_support.check_py3k_warnings(): -# self._getslice_helper_deprecated(Empty) - -# def test_sequence_repeat(self): -# self.assertRaises(OverflowError, lambda: "a" * self.pos) -# self.assertRaises(OverflowError, lambda: "a" * self.neg) - - -# def test_main(): -# support.run_unittest( -# BaseTestCase, -# ListTestCase, -# TupleTestCase, -# # ByteArrayTestCase, -# StringTestCase, -# # UnicodeTestCase, -# # ClassicSeqTestCase, -# # NewSeqTestCase, -# # XRangeTestCase, -# # OverflowTestCase, -# ) - # with test_support.check_py3k_warnings(): - # test_support.run_unittest( - # ClassicSeqDeprecatedTestCase, - # NewSeqDeprecatedTestCase, - # ) + + +class RangeTestCase(unittest.TestCase): + + def test_range(self): + n = newstyle() + n.ind = 5 + self.assertEqual(range(1, 20)[n], 6) + self.assertEqual(range(1, 20).__getitem__(n), 6) + + +class OverflowTestCase(unittest.TestCase): + + def setUp(self): + self.pos = 2**100 + self.neg = -self.pos + + def test_large_longs(self): + self.assertEqual(self.pos.__index__(), self.pos) + self.assertEqual(self.neg.__index__(), self.neg) + + def test_getitem(self): + class GetItem: + def __len__(self): + assert False, "__len__ should not be invoked" + def __getitem__(self, key): + return key + x = GetItem() + self.assertEqual(x[self.pos], self.pos) + self.assertEqual(x[self.neg], self.neg) + self.assertEqual(x[self.neg:self.pos].indices(maxsize), + (0, maxsize, 1)) + self.assertEqual(x[self.neg:self.pos:1].indices(maxsize), + (0, maxsize, 1)) + + def test_sequence_repeat(self): + self.assertRaises(OverflowError, lambda: "a" * self.pos) + self.assertRaises(OverflowError, lambda: "a" * self.neg) if __name__ == "__main__": diff --git a/test/unit3/test_int.py b/test/unit3/test_int.py index ecaec56073..ad0137e26e 100644 --- a/test/unit3/test_int.py +++ b/test/unit3/test_int.py @@ -398,6 +398,21 @@ def __int__(self): # # self.assertRaises(TypeError, lambda: int(TruncReturnsBadInt())) + def test_int_subclass_with_index(self): + class MyIndex(int): + def __index__(self): + return 42 + + class BadIndex(int): + def __index__(self): + return 42.0 + + my_int = MyIndex(7) + self.assertEqual(my_int, 7) + self.assertEqual(int(my_int), 7) + + self.assertEqual(int(BadIndex()), 0) + def test_int_subclass_with_int(self): class MyInt(int): def __int__(self): @@ -414,37 +429,66 @@ def __int__(self): self.assertRaises(TypeError, int, BadInt()) def test_int_returns_int_subclass(self): + class BadIndex: + def __index__(self): + return True + + class BadIndex2(int): + def __index__(self): + return True + class BadInt: def __int__(self): return True - # class BadInt2(int): - # def __int__(self): - # return True + class BadInt2(int): + def __int__(self): + return True + + class TruncReturnsBadIndex: + def __trunc__(self): + return BadIndex() - # class TruncReturnsBadInt: - # def __trunc__(self): - # return BadInt() + class TruncReturnsBadInt: + def __trunc__(self): + return BadInt() class TruncReturnsIntSubclass: def __trunc__(self): return True - bad_int = BadInt() - # self.assertWarns(DeprecationWarning, lambda: int(bad_int)) + bad_int = BadIndex() + # with self.assertWarns(DeprecationWarning): + n = int(bad_int) + self.assertEqual(n, 1) + self.assertIs(type(n), int) + + bad_int = BadIndex2() + n = int(bad_int) + self.assertEqual(n, 0) + self.assertIs(type(n), int) + + # bad_int = BadInt() + # with self.assertWarns(DeprecationWarning): + # n = int(bad_int) + # self.assertEqual(n, 1) + # self.assertIs(type(n), int) # bad_int = BadInt2() - # self.assertWarns(DeprecationWarning, lambda: int(bad_int)) - # n = int(bad_int) + # with self.assertWarns(DeprecationWarning): + # n = int(bad_int) # self.assertEqual(n, 1) # self.assertIs(type(n), int) - # bad_int = TruncReturnsBadInt() - # self.assertWarns(DeprecationWarning, lambda: int(bad_int)) + # bad_int = TruncReturnsBadIndex() + # # with self.assertWarns(DeprecationWarning): # n = int(bad_int) # self.assertEqual(n, 1) # self.assertIs(type(n), int) + # bad_int = TruncReturnsBadInt() + # self.assertRaises(TypeError, int, bad_int) + good_int = TruncReturnsIntSubclass() n = int(good_int) self.assertEqual(n, 1) From 18cf376fdac64b25fcb24dd9d4bc2da178855de8 Mon Sep 17 00:00:00 2001 From: stu Date: Fri, 18 Nov 2022 21:57:58 +0800 Subject: [PATCH 079/137] Add comments about sk$int flag --- src/int.js | 1 + src/misceval.js | 1 + 2 files changed, 2 insertions(+) diff --git a/src/int.js b/src/int.js index 1a242fdc1a..3bfcc322f4 100644 --- a/src/int.js +++ b/src/int.js @@ -421,6 +421,7 @@ Sk.builtin.int_ = Sk.abstr.buildNativeClass("int", { valueOf() { return this.v; }, + // flag to determine inheritance of ints without instanceof sk$int: true, }, }); diff --git a/src/misceval.js b/src/misceval.js index c0895cd4ed..dfbd6abb29 100644 --- a/src/misceval.js +++ b/src/misceval.js @@ -93,6 +93,7 @@ function asIndex(index) { if (index === null || index === undefined) { return; } else if (index.sk$int === true) { + // if we're an int or int subclass use the internal value - as per CPython return index.v; } else if (index.nb$index !== undefined) { return index.nb$index(); // this slot will check the return value is a number / JSBI.BigInt. From 9f3b629406e3f33ba770a731cf0ae373495125b3 Mon Sep 17 00:00:00 2001 From: stu Date: Fri, 18 Nov 2022 22:15:05 +0800 Subject: [PATCH 080/137] fix bug with __delitem__ picked up in the tests after rebase --- src/slotdefs.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/slotdefs.js b/src/slotdefs.js index 1169a8eae1..70af067c9f 100644 --- a/src/slotdefs.js +++ b/src/slotdefs.js @@ -979,7 +979,17 @@ slots.__delitem__ = { $name: "__delitem__", $slot_name: "mp$ass_subscript", $slot_func: slots.__setitem__.$slot_func, - $wrapper: wrapperCallOneArgSuspend, + $wrapper: function (self, args, kwargs) { + // this = the wrapped function + Sk.abstr.checkOneArg(this.$name, args, kwargs); + const res = this.call(self, args[0], undefined, true); + return Sk.misceval.chain(res, (res) => { + if (res === undefined) { + return Sk.builtin.none.none$; + } + return res; + }); + }, $textsig: "($self, key, /)", $flags: { OneArg: true }, $doc: "Delete self[key].", From f098b52e74b34a69845db18364586c250f4f0526 Mon Sep 17 00:00:00 2001 From: stu Date: Fri, 18 Nov 2022 22:30:36 +0800 Subject: [PATCH 081/137] Minor tweak to the __delitem__ change - reuse code for __delete__ --- src/slotdefs.js | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/slotdefs.js b/src/slotdefs.js index 70af067c9f..6f13f7ffdc 100644 --- a/src/slotdefs.js +++ b/src/slotdefs.js @@ -109,6 +109,18 @@ function wrapperSet(self, args, kwargs) { return Sk.misceval.chain(this.call(self, args[0], args[1], true), () => Sk.builtin.none.none$); } +function wrapperDel(self, args, kwargs) { + // this = the wrapped function + Sk.abstr.checkOneArg(this.$name, args, kwargs); + const res = this.call(self, args[0], undefined, true); + return Sk.misceval.chain(res, (res) => { + if (res === undefined) { + return Sk.builtin.none.none$; + } + return res; + }); +} + /** * @param {*} self * @param {Array} args @@ -638,7 +650,7 @@ slots.__delete__ = { $name: "__delete__", $slot_name: "tp$descr_set", $slot_func: slots.__set__.$slot_func, - $wrapper: wrapperCallOneArg, + $wrapper: wrapperDel, $textsig: "($self, instance, /)", $flags: { OneArg: true }, $doc: "Delete an attribute of instance.", @@ -979,17 +991,7 @@ slots.__delitem__ = { $name: "__delitem__", $slot_name: "mp$ass_subscript", $slot_func: slots.__setitem__.$slot_func, - $wrapper: function (self, args, kwargs) { - // this = the wrapped function - Sk.abstr.checkOneArg(this.$name, args, kwargs); - const res = this.call(self, args[0], undefined, true); - return Sk.misceval.chain(res, (res) => { - if (res === undefined) { - return Sk.builtin.none.none$; - } - return res; - }); - }, + $wrapper: wrapperDel, $textsig: "($self, key, /)", $flags: { OneArg: true }, $doc: "Delete self[key].", From 16461f97a7a9bf5c08f9de05e5a2463b5b22da6b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 18 Nov 2022 14:36:08 +0000 Subject: [PATCH 082/137] build(deps): bump loader-utils from 1.4.0 to 1.4.2 Bumps [loader-utils](https://github.com/webpack/loader-utils) from 1.4.0 to 1.4.2. - [Release notes](https://github.com/webpack/loader-utils/releases) - [Changelog](https://github.com/webpack/loader-utils/blob/v1.4.2/CHANGELOG.md) - [Commits](https://github.com/webpack/loader-utils/compare/v1.4.0...v1.4.2) --- updated-dependencies: - dependency-name: loader-utils dependency-type: indirect ... Signed-off-by: dependabot[bot] --- package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index feceba924d..d0639ec3e3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3883,9 +3883,9 @@ } }, "node_modules/loader-utils": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-1.4.0.tgz", - "integrity": "sha512-qH0WSMBtn/oHuwjy/NucEgbx5dbxxnxup9s4PVXJUDHZBQY+s0NWA9rJf53RBnQZxfch7euUui7hpoAPvALZdA==", + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-1.4.2.tgz", + "integrity": "sha512-I5d00Pd/jwMD2QCduo657+YM/6L3KZu++pmX9VFncxaxvHcru9jx1lBaFft+r4Mt2jK0Yhp41XlRAihzPxHNCg==", "dev": true, "dependencies": { "big.js": "^5.2.2", @@ -10127,9 +10127,9 @@ "dev": true }, "loader-utils": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-1.4.0.tgz", - "integrity": "sha512-qH0WSMBtn/oHuwjy/NucEgbx5dbxxnxup9s4PVXJUDHZBQY+s0NWA9rJf53RBnQZxfch7euUui7hpoAPvALZdA==", + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-1.4.2.tgz", + "integrity": "sha512-I5d00Pd/jwMD2QCduo657+YM/6L3KZu++pmX9VFncxaxvHcru9jx1lBaFft+r4Mt2jK0Yhp41XlRAihzPxHNCg==", "dev": true, "requires": { "big.js": "^5.2.2", From fae0e813e8234d5a7e2bda60f1bb54ab9980708b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 18 Nov 2022 14:43:43 +0000 Subject: [PATCH 083/137] build(deps-dev): bump terser from 5.7.0 to 5.14.2 Bumps [terser](https://github.com/terser/terser) from 5.7.0 to 5.14.2. - [Release notes](https://github.com/terser/terser/releases) - [Changelog](https://github.com/terser/terser/blob/master/CHANGELOG.md) - [Commits](https://github.com/terser/terser/commits) --- updated-dependencies: - dependency-name: terser dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- package-lock.json | 164 ++++++++++++++++++++++++++++++++++++++-------- package.json | 2 +- 2 files changed, 139 insertions(+), 27 deletions(-) diff --git a/package-lock.json b/package-lock.json index d0639ec3e3..3870462187 100644 --- a/package-lock.json +++ b/package-lock.json @@ -34,7 +34,7 @@ "setimmediate": "^1.0.5", "shelljs": "^0.8.5", "strftime": "^0.10.1", - "terser": "^5.7.0", + "terser": "^5.14.2", "webpack": "^4.32.0", "webpack-cli": "^3.3.2" }, @@ -80,6 +80,64 @@ "node": ">=6.0.0" } }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz", + "integrity": "sha512-mh65xKQAzI6iBcFzwv28KVWSmCkdRBWoOh+bYQGW3+6OZvbbN3TqMGo5hqYxQniRcH9F2VZIoJCm4pa3BPDK/A==", + "dev": true, + "dependencies": { + "@jridgewell/set-array": "^1.0.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.9" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", + "integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/set-array": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz", + "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/source-map": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.2.tgz", + "integrity": "sha512-m7O9o2uR8k2ObDysZYzdfhb08VuEml5oWGiosa1VdaPZ/A6QyPkAJuwN0Q1lhULOf6B7MtQmHENS743hWtCrgw==", + "dev": true, + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.0", + "@jridgewell/trace-mapping": "^0.3.9" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.4.14", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", + "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==", + "dev": true + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.17", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.17.tgz", + "integrity": "sha512-MCNzAp77qzKca9+W/+I0+sEpaUnZoeasnghNeVc41VZCEKaCH73Vq3BZZ/SzWIgrqE4H4ceI+p+b6C0mHf9T4g==", + "dev": true, + "dependencies": { + "@jridgewell/resolve-uri": "3.1.0", + "@jridgewell/sourcemap-codec": "1.4.14" + } + }, "node_modules/@types/glob": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.1.3.tgz", @@ -5684,9 +5742,9 @@ } }, "node_modules/source-map-support": { - "version": "0.5.19", - "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz", - "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==", + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", "dev": true, "dependencies": { "buffer-from": "^1.0.0", @@ -5938,14 +5996,15 @@ } }, "node_modules/terser": { - "version": "5.7.0", - "resolved": "https://registry.npmjs.org/terser/-/terser-5.7.0.tgz", - "integrity": "sha512-HP5/9hp2UaZt5fYkuhNBR8YyRcT8juw8+uFbAme53iN9hblvKnLUTKkmwJG6ocWpIKf8UK4DoeWG4ty0J6S6/g==", + "version": "5.14.2", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.14.2.tgz", + "integrity": "sha512-oL0rGeM/WFQCUd0y2QrWxYnq7tfSuKBiqTjRPWrRgB46WD/kiwHwF8T23z78H6Q6kGCuuHcPB+KULHRdxvVGQA==", "dev": true, "dependencies": { + "@jridgewell/source-map": "^0.3.2", + "acorn": "^8.5.0", "commander": "^2.20.0", - "source-map": "~0.7.2", - "source-map-support": "~0.5.19" + "source-map-support": "~0.5.20" }, "bin": { "terser": "bin/terser" @@ -6032,13 +6091,16 @@ "node": ">=6.0.0" } }, - "node_modules/terser/node_modules/source-map": { - "version": "0.7.3", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz", - "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==", + "node_modules/terser/node_modules/acorn": { + "version": "8.8.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.1.tgz", + "integrity": "sha512-7zFpHzhnqYKrkYdUjF1HI1bzd0VygEGX8lFk4k5zVMqHEoES+P+7TKI+EvLO9WVMJ8eekdO0aDEK044xTXwPPA==", "dev": true, + "bin": { + "acorn": "bin/acorn" + }, "engines": { - "node": ">= 8" + "node": ">=0.4.0" } }, "node_modules/text-table": { @@ -6907,6 +6969,55 @@ "integrity": "sha512-r1yKkiUTYMQ8LiEI0UcQx5ETw5dpTLn9wijn9hk6KkTtOK95FndDN10M+8/s6k/Ymlbivw0Av9q4SlgF80PtHg==", "dev": true }, + "@jridgewell/gen-mapping": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz", + "integrity": "sha512-mh65xKQAzI6iBcFzwv28KVWSmCkdRBWoOh+bYQGW3+6OZvbbN3TqMGo5hqYxQniRcH9F2VZIoJCm4pa3BPDK/A==", + "dev": true, + "requires": { + "@jridgewell/set-array": "^1.0.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.9" + } + }, + "@jridgewell/resolve-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", + "integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==", + "dev": true + }, + "@jridgewell/set-array": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz", + "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==", + "dev": true + }, + "@jridgewell/source-map": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.2.tgz", + "integrity": "sha512-m7O9o2uR8k2ObDysZYzdfhb08VuEml5oWGiosa1VdaPZ/A6QyPkAJuwN0Q1lhULOf6B7MtQmHENS743hWtCrgw==", + "dev": true, + "requires": { + "@jridgewell/gen-mapping": "^0.3.0", + "@jridgewell/trace-mapping": "^0.3.9" + } + }, + "@jridgewell/sourcemap-codec": { + "version": "1.4.14", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", + "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==", + "dev": true + }, + "@jridgewell/trace-mapping": { + "version": "0.3.17", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.17.tgz", + "integrity": "sha512-MCNzAp77qzKca9+W/+I0+sEpaUnZoeasnghNeVc41VZCEKaCH73Vq3BZZ/SzWIgrqE4H4ceI+p+b6C0mHf9T4g==", + "dev": true, + "requires": { + "@jridgewell/resolve-uri": "3.1.0", + "@jridgewell/sourcemap-codec": "1.4.14" + } + }, "@types/glob": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.1.3.tgz", @@ -11621,9 +11732,9 @@ } }, "source-map-support": { - "version": "0.5.19", - "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz", - "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==", + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", "dev": true, "requires": { "buffer-from": "^1.0.0", @@ -11836,20 +11947,21 @@ "dev": true }, "terser": { - "version": "5.7.0", - "resolved": "https://registry.npmjs.org/terser/-/terser-5.7.0.tgz", - "integrity": "sha512-HP5/9hp2UaZt5fYkuhNBR8YyRcT8juw8+uFbAme53iN9hblvKnLUTKkmwJG6ocWpIKf8UK4DoeWG4ty0J6S6/g==", + "version": "5.14.2", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.14.2.tgz", + "integrity": "sha512-oL0rGeM/WFQCUd0y2QrWxYnq7tfSuKBiqTjRPWrRgB46WD/kiwHwF8T23z78H6Q6kGCuuHcPB+KULHRdxvVGQA==", "dev": true, "requires": { + "@jridgewell/source-map": "^0.3.2", + "acorn": "^8.5.0", "commander": "^2.20.0", - "source-map": "~0.7.2", - "source-map-support": "~0.5.19" + "source-map-support": "~0.5.20" }, "dependencies": { - "source-map": { - "version": "0.7.3", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz", - "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==", + "acorn": { + "version": "8.8.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.1.tgz", + "integrity": "sha512-7zFpHzhnqYKrkYdUjF1HI1bzd0VygEGX8lFk4k5zVMqHEoES+P+7TKI+EvLO9WVMJ8eekdO0aDEK044xTXwPPA==", "dev": true } } diff --git a/package.json b/package.json index 122073dac1..9f18bcad87 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,7 @@ "setimmediate": "^1.0.5", "shelljs": "^0.8.5", "strftime": "^0.10.1", - "terser": "^5.7.0", + "terser": "^5.14.2", "webpack": "^4.32.0", "webpack-cli": "^3.3.2" }, From 97f2dc8b8c4b3b72eb081b1b95c243c42589fc2d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 18 Nov 2022 14:46:27 +0000 Subject: [PATCH 084/137] build(deps): bump minimatch from 3.0.4 to 3.1.2 Bumps [minimatch](https://github.com/isaacs/minimatch) from 3.0.4 to 3.1.2. - [Release notes](https://github.com/isaacs/minimatch/releases) - [Commits](https://github.com/isaacs/minimatch/compare/v3.0.4...v3.1.2) --- updated-dependencies: - dependency-name: minimatch dependency-type: indirect ... Signed-off-by: dependabot[bot] --- package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3870462187..2e9d1239b0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4217,9 +4217,9 @@ "dev": true }, "node_modules/minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "dev": true, "dependencies": { "brace-expansion": "^1.1.7" @@ -10464,9 +10464,9 @@ "dev": true }, "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "dev": true, "requires": { "brace-expansion": "^1.1.7" From 4892ca9e38fd2c9fe42c0374fa281d25cafe6f1c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 18 Nov 2022 14:50:25 +0000 Subject: [PATCH 085/137] build(deps): bump markdown-it and jsdoc Bumps [markdown-it](https://github.com/markdown-it/markdown-it) to 12.3.2 and updates ancestor dependency [jsdoc](https://github.com/jsdoc/jsdoc). These dependencies need to be updated together. Updates `markdown-it` from 10.0.0 to 12.3.2 - [Release notes](https://github.com/markdown-it/markdown-it/releases) - [Changelog](https://github.com/markdown-it/markdown-it/blob/master/CHANGELOG.md) - [Commits](https://github.com/markdown-it/markdown-it/compare/10.0.0...12.3.2) Updates `jsdoc` from 3.6.6 to 3.6.11 - [Release notes](https://github.com/jsdoc/jsdoc/releases) - [Changelog](https://github.com/jsdoc/jsdoc/blob/main/CHANGES.md) - [Commits](https://github.com/jsdoc/jsdoc/compare/3.6.6...3.6.11) --- updated-dependencies: - dependency-name: markdown-it dependency-type: indirect - dependency-name: jsdoc dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- package-lock.json | 250 +++++++++++++++++++++++++++++----------------- package.json | 2 +- 2 files changed, 160 insertions(+), 92 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2e9d1239b0..09532e4fa2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,7 +27,7 @@ "git-revision-webpack-plugin": "^3.0.3", "google-closure-compiler": "^20210202.0.0", "js-beautify": "^1.10.0", - "jsdoc": "^3.6.3", + "jsdoc": "^3.6.11", "micro-strptime": "^0.2.3", "open": "^6.3.0", "readline-sync": "^1.4.9", @@ -148,6 +148,28 @@ "@types/node": "*" } }, + "node_modules/@types/linkify-it": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/linkify-it/-/linkify-it-3.0.2.tgz", + "integrity": "sha512-HZQYqbiFVWufzCwexrvh694SOim8z2d+xJl5UNamcvQFejLY/2YUtzXHYi3cHdI7PMlS8ejH2slRAOJQ32aNbA==", + "dev": true + }, + "node_modules/@types/markdown-it": { + "version": "12.2.3", + "resolved": "https://registry.npmjs.org/@types/markdown-it/-/markdown-it-12.2.3.tgz", + "integrity": "sha512-GKMHFfv3458yYy+v/N8gjufHO6MSZKCOXpZc5GXIWWy8uldwfmPn98vp81gZ5f9SVw8YYBctgfJ22a2d7AOMeQ==", + "dev": true, + "dependencies": { + "@types/linkify-it": "*", + "@types/mdurl": "*" + } + }, + "node_modules/@types/mdurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@types/mdurl/-/mdurl-1.0.2.tgz", + "integrity": "sha512-eC4U9MlIcu2q0KQmXszyn5Akca/0jrQmwDRgpAMJai7qBWq4amIQhZyNau4VYGtCeALvW1/NtjzJJ567aZxfKA==", + "dev": true + }, "node_modules/@types/minimatch": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz", @@ -1025,15 +1047,15 @@ } }, "node_modules/catharsis": { - "version": "0.8.11", - "resolved": "https://registry.npmjs.org/catharsis/-/catharsis-0.8.11.tgz", - "integrity": "sha512-a+xUyMV7hD1BrDQA/3iPV7oc+6W26BgVJO05PGEoatMyIuPScQKsde6i3YorWX1qs+AZjnJ18NqdKoCtKiNh1g==", + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/catharsis/-/catharsis-0.9.0.tgz", + "integrity": "sha512-prMTQVpcns/tzFgFVkVp6ak6RykZyWb3gu8ckUpd6YkTlacOd3DXGJjIpD4Q6zJirizvaiAjSSHlOsA+6sNh2A==", "dev": true, "dependencies": { - "lodash": "^4.17.14" + "lodash": "^4.17.15" }, "engines": { - "node": ">= 8" + "node": ">= 10" } }, "node_modules/chalk": { @@ -2004,10 +2026,13 @@ } }, "node_modules/entities": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/entities/-/entities-2.0.3.tgz", - "integrity": "sha512-MyoZ0jgnLvB2X3Lg5HqpFmn1kybDiIfEQmKzTb5apr51Rb+T3KdmMiqa70T+bhGnyv7bQ6WMj2QMHpGMmlrUYQ==", - "dev": true + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-2.1.0.tgz", + "integrity": "sha512-hCx1oky9PFrJ611mf0ifBLBRW8lUUVRlFolb5gWRfIELabBlbp9xZvrqZLZAs+NxFnbfQoeGd8wDkygjg7U85w==", + "dev": true, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } }, "node_modules/errno": { "version": "0.1.8", @@ -3741,12 +3766,12 @@ } }, "node_modules/js2xmlparser": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/js2xmlparser/-/js2xmlparser-4.0.1.tgz", - "integrity": "sha512-KrPTolcw6RocpYjdC7pL7v62e55q7qOMHvLX1UCLc5AAS8qeJ6nukarEJAF2KL2PZxlbGueEbINqZR2bDe/gUw==", + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/js2xmlparser/-/js2xmlparser-4.0.2.tgz", + "integrity": "sha512-6n4D8gLlLf1n5mNLQPRfViYzu9RATblzPEtm1SthMX1Pjao0r9YI9nw7ZIfRxQMERS87mcswrg+r/OYrPRX6jA==", "dev": true, "dependencies": { - "xmlcreate": "^2.0.3" + "xmlcreate": "^2.0.4" } }, "node_modules/jsbi": { @@ -3755,31 +3780,32 @@ "integrity": "sha512-j6G1XZPxyYxALX0XvgJMkyfwCpG/NOmzS8vDwXcAO5y3l2aclpabU9W0yli+G7Bfpv4j9Sg+Ln70i+wENNcaaw==" }, "node_modules/jsdoc": { - "version": "3.6.6", - "resolved": "https://registry.npmjs.org/jsdoc/-/jsdoc-3.6.6.tgz", - "integrity": "sha512-znR99e1BHeyEkSvgDDpX0sTiTu+8aQyDl9DawrkOGZTTW8hv0deIFXx87114zJ7gRaDZKVQD/4tr1ifmJp9xhQ==", + "version": "3.6.11", + "resolved": "https://registry.npmjs.org/jsdoc/-/jsdoc-3.6.11.tgz", + "integrity": "sha512-8UCU0TYeIYD9KeLzEcAu2q8N/mx9O3phAGl32nmHlE0LpaJL71mMkP4d+QE5zWfNt50qheHtOZ0qoxVrsX5TUg==", "dev": true, "dependencies": { "@babel/parser": "^7.9.4", + "@types/markdown-it": "^12.2.3", "bluebird": "^3.7.2", - "catharsis": "^0.8.11", + "catharsis": "^0.9.0", "escape-string-regexp": "^2.0.0", - "js2xmlparser": "^4.0.1", + "js2xmlparser": "^4.0.2", "klaw": "^3.0.0", - "markdown-it": "^10.0.0", - "markdown-it-anchor": "^5.2.7", - "marked": "^0.8.2", + "markdown-it": "^12.3.2", + "markdown-it-anchor": "^8.4.1", + "marked": "^4.0.10", "mkdirp": "^1.0.4", "requizzle": "^0.2.3", "strip-json-comments": "^3.1.0", "taffydb": "2.6.2", - "underscore": "~1.10.2" + "underscore": "~1.13.2" }, "bin": { "jsdoc": "jsdoc.js" }, "engines": { - "node": ">=8.15.0" + "node": ">=12.0.0" } }, "node_modules/jsdoc/node_modules/escape-string-regexp": { @@ -3862,9 +3888,9 @@ } }, "node_modules/linkify-it": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-2.2.0.tgz", - "integrity": "sha512-GnAl/knGn+i1U/wjBz3akz2stz+HrHLsxMwHQGofCDfPvlf+gDKN58UtfmUquTY4/MXeE2x7k19KQmeoZi94Iw==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-3.0.3.tgz", + "integrity": "sha512-ynTsyrFSdE5oZ/O9GEf00kPngmOfVwazR5GKDq6EYfhlpFug3J2zybX56a2PRRpc9P+FuSoGNAwjlbDs9jJBPQ==", "dev": true, "dependencies": { "uc.micro": "^1.0.1" @@ -4029,14 +4055,14 @@ } }, "node_modules/markdown-it": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-10.0.0.tgz", - "integrity": "sha512-YWOP1j7UbDNz+TumYP1kpwnP0aEa711cJjrAQrzd0UXlbJfc5aAq0F/PZHjiioqDC1NKgvIMX+o+9Bk7yuM2dg==", + "version": "12.3.2", + "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-12.3.2.tgz", + "integrity": "sha512-TchMembfxfNVpHkbtriWltGWc+m3xszaRD0CZup7GFFhzIgQqxIfn3eGj1yZpfuflzPvfkt611B2Q/Bsk1YnGg==", "dev": true, "dependencies": { - "argparse": "^1.0.7", - "entities": "~2.0.0", - "linkify-it": "^2.0.0", + "argparse": "^2.0.1", + "entities": "~2.1.0", + "linkify-it": "^3.0.1", "mdurl": "^1.0.1", "uc.micro": "^1.0.5" }, @@ -4045,21 +4071,31 @@ } }, "node_modules/markdown-it-anchor": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/markdown-it-anchor/-/markdown-it-anchor-5.3.0.tgz", - "integrity": "sha512-/V1MnLL/rgJ3jkMWo84UR+K+jF1cxNG1a+KwqeXqTIJ+jtA8aWSHuigx8lTzauiIjBDbwF3NcWQMotd0Dm39jA==", + "version": "8.6.5", + "resolved": "https://registry.npmjs.org/markdown-it-anchor/-/markdown-it-anchor-8.6.5.tgz", + "integrity": "sha512-PI1qEHHkTNWT+X6Ip9w+paonfIQ+QZP9sCeMYi47oqhH+EsW8CrJ8J7CzV19QVOj6il8ATGbK2nTECj22ZHGvQ==", + "dev": true, + "peerDependencies": { + "@types/markdown-it": "*", + "markdown-it": "*" + } + }, + "node_modules/markdown-it/node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "dev": true }, "node_modules/marked": { - "version": "0.8.2", - "resolved": "https://registry.npmjs.org/marked/-/marked-0.8.2.tgz", - "integrity": "sha512-EGwzEeCcLniFX51DhTpmTom+dSA/MG/OBUDjnWtHbEnjAH180VzUeAw+oE4+Zv+CoYBWyRlYOTR0N8SO9R1PVw==", + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/marked/-/marked-4.2.2.tgz", + "integrity": "sha512-JjBTFTAvuTgANXx82a5vzK9JLSMoV6V3LBVn4Uhdso6t7vXrGx7g1Cd2r6NYSsxrYbQGFCMqBDhFHyK5q2UvcQ==", "dev": true, "bin": { - "marked": "bin/marked" + "marked": "bin/marked.js" }, "engines": { - "node": ">= 8.16.2" + "node": ">= 12" } }, "node_modules/md5.js": { @@ -4076,7 +4112,7 @@ "node_modules/mdurl": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-1.0.1.tgz", - "integrity": "sha1-/oWy7HWlkDfyrf7BAP1sYBdhFS4=", + "integrity": "sha512-/sKlQJCBYVY9Ers9hqzKou4H6V5UWc/M59TH2dvkt+84itfnq7uFOMLpOiOS4ujvHP4etln18fmIxA5R5fll0g==", "dev": true }, "node_modules/media-typer": { @@ -6266,9 +6302,9 @@ "dev": true }, "node_modules/underscore": { - "version": "1.10.2", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.10.2.tgz", - "integrity": "sha512-N4P+Q/BuyuEKFJ43B9gYuOj4TQUHXX+j2FqguVOpjkssLUUrnJofCcBccJSCoeturDoZU6GorDTHSvUDlSQbTg==", + "version": "1.13.6", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.6.tgz", + "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==", "dev": true }, "node_modules/union-value": { @@ -6837,9 +6873,9 @@ } }, "node_modules/xmlcreate": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/xmlcreate/-/xmlcreate-2.0.3.tgz", - "integrity": "sha512-HgS+X6zAztGa9zIK3Y3LXuJes33Lz9x+YyTxgrkIdabu2vqcGOWwdfCpf1hWLRrd553wd4QCDf6BBO6FfdsRiQ==", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/xmlcreate/-/xmlcreate-2.0.4.tgz", + "integrity": "sha512-nquOebG4sngPmGPICTS5EnxqhKbCmz5Ox5hsszI2T6U5qdrJizBc+0ilYSEjTSzU0yZcmvppztXe/5Al5fUwdg==", "dev": true }, "node_modules/xtend": { @@ -7028,6 +7064,28 @@ "@types/node": "*" } }, + "@types/linkify-it": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/linkify-it/-/linkify-it-3.0.2.tgz", + "integrity": "sha512-HZQYqbiFVWufzCwexrvh694SOim8z2d+xJl5UNamcvQFejLY/2YUtzXHYi3cHdI7PMlS8ejH2slRAOJQ32aNbA==", + "dev": true + }, + "@types/markdown-it": { + "version": "12.2.3", + "resolved": "https://registry.npmjs.org/@types/markdown-it/-/markdown-it-12.2.3.tgz", + "integrity": "sha512-GKMHFfv3458yYy+v/N8gjufHO6MSZKCOXpZc5GXIWWy8uldwfmPn98vp81gZ5f9SVw8YYBctgfJ22a2d7AOMeQ==", + "dev": true, + "requires": { + "@types/linkify-it": "*", + "@types/mdurl": "*" + } + }, + "@types/mdurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@types/mdurl/-/mdurl-1.0.2.tgz", + "integrity": "sha512-eC4U9MlIcu2q0KQmXszyn5Akca/0jrQmwDRgpAMJai7qBWq4amIQhZyNau4VYGtCeALvW1/NtjzJJ567aZxfKA==", + "dev": true + }, "@types/minimatch": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz", @@ -7817,12 +7875,12 @@ "dev": true }, "catharsis": { - "version": "0.8.11", - "resolved": "https://registry.npmjs.org/catharsis/-/catharsis-0.8.11.tgz", - "integrity": "sha512-a+xUyMV7hD1BrDQA/3iPV7oc+6W26BgVJO05PGEoatMyIuPScQKsde6i3YorWX1qs+AZjnJ18NqdKoCtKiNh1g==", + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/catharsis/-/catharsis-0.9.0.tgz", + "integrity": "sha512-prMTQVpcns/tzFgFVkVp6ak6RykZyWb3gu8ckUpd6YkTlacOd3DXGJjIpD4Q6zJirizvaiAjSSHlOsA+6sNh2A==", "dev": true, "requires": { - "lodash": "^4.17.14" + "lodash": "^4.17.15" } }, "chalk": { @@ -8661,9 +8719,9 @@ } }, "entities": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/entities/-/entities-2.0.3.tgz", - "integrity": "sha512-MyoZ0jgnLvB2X3Lg5HqpFmn1kybDiIfEQmKzTb5apr51Rb+T3KdmMiqa70T+bhGnyv7bQ6WMj2QMHpGMmlrUYQ==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-2.1.0.tgz", + "integrity": "sha512-hCx1oky9PFrJ611mf0ifBLBRW8lUUVRlFolb5gWRfIELabBlbp9xZvrqZLZAs+NxFnbfQoeGd8wDkygjg7U85w==", "dev": true }, "errno": { @@ -10073,12 +10131,12 @@ } }, "js2xmlparser": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/js2xmlparser/-/js2xmlparser-4.0.1.tgz", - "integrity": "sha512-KrPTolcw6RocpYjdC7pL7v62e55q7qOMHvLX1UCLc5AAS8qeJ6nukarEJAF2KL2PZxlbGueEbINqZR2bDe/gUw==", + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/js2xmlparser/-/js2xmlparser-4.0.2.tgz", + "integrity": "sha512-6n4D8gLlLf1n5mNLQPRfViYzu9RATblzPEtm1SthMX1Pjao0r9YI9nw7ZIfRxQMERS87mcswrg+r/OYrPRX6jA==", "dev": true, "requires": { - "xmlcreate": "^2.0.3" + "xmlcreate": "^2.0.4" } }, "jsbi": { @@ -10087,25 +10145,26 @@ "integrity": "sha512-j6G1XZPxyYxALX0XvgJMkyfwCpG/NOmzS8vDwXcAO5y3l2aclpabU9W0yli+G7Bfpv4j9Sg+Ln70i+wENNcaaw==" }, "jsdoc": { - "version": "3.6.6", - "resolved": "https://registry.npmjs.org/jsdoc/-/jsdoc-3.6.6.tgz", - "integrity": "sha512-znR99e1BHeyEkSvgDDpX0sTiTu+8aQyDl9DawrkOGZTTW8hv0deIFXx87114zJ7gRaDZKVQD/4tr1ifmJp9xhQ==", + "version": "3.6.11", + "resolved": "https://registry.npmjs.org/jsdoc/-/jsdoc-3.6.11.tgz", + "integrity": "sha512-8UCU0TYeIYD9KeLzEcAu2q8N/mx9O3phAGl32nmHlE0LpaJL71mMkP4d+QE5zWfNt50qheHtOZ0qoxVrsX5TUg==", "dev": true, "requires": { "@babel/parser": "^7.9.4", + "@types/markdown-it": "^12.2.3", "bluebird": "^3.7.2", - "catharsis": "^0.8.11", + "catharsis": "^0.9.0", "escape-string-regexp": "^2.0.0", - "js2xmlparser": "^4.0.1", + "js2xmlparser": "^4.0.2", "klaw": "^3.0.0", - "markdown-it": "^10.0.0", - "markdown-it-anchor": "^5.2.7", - "marked": "^0.8.2", + "markdown-it": "^12.3.2", + "markdown-it-anchor": "^8.4.1", + "marked": "^4.0.10", "mkdirp": "^1.0.4", "requizzle": "^0.2.3", "strip-json-comments": "^3.1.0", "taffydb": "2.6.2", - "underscore": "~1.10.2" + "underscore": "~1.13.2" }, "dependencies": { "escape-string-regexp": { @@ -10172,9 +10231,9 @@ } }, "linkify-it": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-2.2.0.tgz", - "integrity": "sha512-GnAl/knGn+i1U/wjBz3akz2stz+HrHLsxMwHQGofCDfPvlf+gDKN58UtfmUquTY4/MXeE2x7k19KQmeoZi94Iw==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-3.0.3.tgz", + "integrity": "sha512-ynTsyrFSdE5oZ/O9GEf00kPngmOfVwazR5GKDq6EYfhlpFug3J2zybX56a2PRRpc9P+FuSoGNAwjlbDs9jJBPQ==", "dev": true, "requires": { "uc.micro": "^1.0.1" @@ -10310,28 +10369,37 @@ } }, "markdown-it": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-10.0.0.tgz", - "integrity": "sha512-YWOP1j7UbDNz+TumYP1kpwnP0aEa711cJjrAQrzd0UXlbJfc5aAq0F/PZHjiioqDC1NKgvIMX+o+9Bk7yuM2dg==", + "version": "12.3.2", + "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-12.3.2.tgz", + "integrity": "sha512-TchMembfxfNVpHkbtriWltGWc+m3xszaRD0CZup7GFFhzIgQqxIfn3eGj1yZpfuflzPvfkt611B2Q/Bsk1YnGg==", "dev": true, "requires": { - "argparse": "^1.0.7", - "entities": "~2.0.0", - "linkify-it": "^2.0.0", + "argparse": "^2.0.1", + "entities": "~2.1.0", + "linkify-it": "^3.0.1", "mdurl": "^1.0.1", "uc.micro": "^1.0.5" + }, + "dependencies": { + "argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true + } } }, "markdown-it-anchor": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/markdown-it-anchor/-/markdown-it-anchor-5.3.0.tgz", - "integrity": "sha512-/V1MnLL/rgJ3jkMWo84UR+K+jF1cxNG1a+KwqeXqTIJ+jtA8aWSHuigx8lTzauiIjBDbwF3NcWQMotd0Dm39jA==", - "dev": true + "version": "8.6.5", + "resolved": "https://registry.npmjs.org/markdown-it-anchor/-/markdown-it-anchor-8.6.5.tgz", + "integrity": "sha512-PI1qEHHkTNWT+X6Ip9w+paonfIQ+QZP9sCeMYi47oqhH+EsW8CrJ8J7CzV19QVOj6il8ATGbK2nTECj22ZHGvQ==", + "dev": true, + "requires": {} }, "marked": { - "version": "0.8.2", - "resolved": "https://registry.npmjs.org/marked/-/marked-0.8.2.tgz", - "integrity": "sha512-EGwzEeCcLniFX51DhTpmTom+dSA/MG/OBUDjnWtHbEnjAH180VzUeAw+oE4+Zv+CoYBWyRlYOTR0N8SO9R1PVw==", + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/marked/-/marked-4.2.2.tgz", + "integrity": "sha512-JjBTFTAvuTgANXx82a5vzK9JLSMoV6V3LBVn4Uhdso6t7vXrGx7g1Cd2r6NYSsxrYbQGFCMqBDhFHyK5q2UvcQ==", "dev": true }, "md5.js": { @@ -10348,7 +10416,7 @@ "mdurl": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-1.0.1.tgz", - "integrity": "sha1-/oWy7HWlkDfyrf7BAP1sYBdhFS4=", + "integrity": "sha512-/sKlQJCBYVY9Ers9hqzKou4H6V5UWc/M59TH2dvkt+84itfnq7uFOMLpOiOS4ujvHP4etln18fmIxA5R5fll0g==", "dev": true }, "media-typer": { @@ -12172,9 +12240,9 @@ "dev": true }, "underscore": { - "version": "1.10.2", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.10.2.tgz", - "integrity": "sha512-N4P+Q/BuyuEKFJ43B9gYuOj4TQUHXX+j2FqguVOpjkssLUUrnJofCcBccJSCoeturDoZU6GorDTHSvUDlSQbTg==", + "version": "1.13.6", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.6.tgz", + "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==", "dev": true }, "union-value": { @@ -12659,9 +12727,9 @@ } }, "xmlcreate": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/xmlcreate/-/xmlcreate-2.0.3.tgz", - "integrity": "sha512-HgS+X6zAztGa9zIK3Y3LXuJes33Lz9x+YyTxgrkIdabu2vqcGOWwdfCpf1hWLRrd553wd4QCDf6BBO6FfdsRiQ==", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/xmlcreate/-/xmlcreate-2.0.4.tgz", + "integrity": "sha512-nquOebG4sngPmGPICTS5EnxqhKbCmz5Ox5hsszI2T6U5qdrJizBc+0ilYSEjTSzU0yZcmvppztXe/5Al5fUwdg==", "dev": true }, "xtend": { diff --git a/package.json b/package.json index 9f18bcad87..6d5084ebe4 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,7 @@ "git-revision-webpack-plugin": "^3.0.3", "google-closure-compiler": "^20210202.0.0", "js-beautify": "^1.10.0", - "jsdoc": "^3.6.3", + "jsdoc": "^3.6.11", "micro-strptime": "^0.2.3", "open": "^6.3.0", "readline-sync": "^1.4.9", From 2f4519fd740638ccde759793d91699ad1d17ad24 Mon Sep 17 00:00:00 2001 From: mrcork Date: Thu, 18 Feb 2021 13:36:12 +0800 Subject: [PATCH 086/137] include a valueOf property for all the builtins that does the obvious thing --- src/abstract.js | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/abstract.js b/src/abstract.js index 2f28d63391..176a0924c8 100644 --- a/src/abstract.js +++ b/src/abstract.js @@ -1333,6 +1333,12 @@ Sk.abstr.buildNativeClass = function (typename, options) { }); }); + + if (typeobject.prototype.hasOwnProperty("tp$iter")) { + typeobject.prototype[Symbol.iterator] = function () { + return this.tp$iter()[Symbol.iterator](); + }; + } // str might not have been created yet if (Sk.builtin.str !== undefined && type_proto.hasOwnProperty("tp$doc") && !type_proto.hasOwnProperty("__doc__")) { const docstr = type_proto.tp$doc || null; @@ -1391,6 +1397,18 @@ Sk.abstr.buildIteratorClass = function (typename, iterator) { iterator.slots.tp$getattr = iterator.slots.tp$getattr || Sk.generic.getAttr; let ret = Sk.abstr.buildNativeClass(typename, iterator); Sk.abstr.built$iterators.push(ret); + + ret.prototype[Symbol.iterator] = function () { + return { + next: () => { + const nxt = this.tp$iternext(); + if (nxt === undefined) { + return {done: true}; + } + return {value: nxt, done: false}; + } + }; + }; return ret; }; From c5b85221e6ff8590a708315b3591700d24778377 Mon Sep 17 00:00:00 2001 From: mrcork Date: Wed, 23 Sep 2020 22:20:56 +0800 Subject: [PATCH 087/137] adjusted ffi api --- src/ffi.js | 802 ++++++++++++++++++++++++++++++++++++++++++-------- src/lib/re.js | 8 +- 2 files changed, 681 insertions(+), 129 deletions(-) diff --git a/src/ffi.js b/src/ffi.js index a3f0283a6f..428112c9a2 100644 --- a/src/ffi.js +++ b/src/ffi.js @@ -2,169 +2,721 @@ * @namespace Sk.ffi * */ +Sk.ffi = { + remapToPy: toPy, + remapToJs: toJs, + toPy, + toJs, -Sk.ffi = Sk.ffi || {}; + isTrue, + + toJsString, + toJsNumber, + toJsArray, + + toJsHashMap, + + toPyDict, + toPyFloat, + toPyInt, + toPyNumber, + toPyStr, + toPyList, + toPyTuple, + toPySet, + + numberToPy, + proxy, +}; /** * maps from Javascript Object/Array/string to Python dict/list/str. * * only works on basic objects that are being used as storage, doesn't handle * functions, etc. + * hooks.funcHook + * hooks.dictHook */ -Sk.ffi.remapToPy = function (obj) { - var k; - var kvs; - var i; - var arr; - - if (obj === null || typeof obj === "undefined") { +function toPy(obj, hooks) { + if (obj === null || obj === undefined) { return Sk.builtin.none.none$; } - if (obj.ob$type) { + if (obj.sk$object) { return obj; + } else if (obj.$isPyWrapped && obj.unwrap) { + // wrap protocol + return obj.unwrap(); + } + + const type = typeof obj; + hooks = hooks || {}; + + if (type === "string") { + return new Sk.builtin.str(obj); + } else if (type === "number") { + return numberToPy(obj); + } else if (type === "boolean") { + return new Sk.builtin.bool(obj); + } else if (type === "function") { + // should the defualt behaviour be to proxy or new Sk.builtin.func? + // old remap used to do an Sk.builtin.func + return hooks.funcHook ? hooks.funcHook(obj) : proxy(obj); + } else if (JSBI.__isBigInt(obj)) { + // might be type === "bigint" if bigint native or an array like object for older browsers + return new Sk.builtin.int_(JSBI.numberIfSafe(obj)); + } else if (Array.isArray(obj)) { + return new Sk.builtin.list(obj.map((x) => toPy(x, hooks))); + } else if (type === "object") { + const constructor = obj.constructor; // it's possible that a library deleted the constructor + if ( + (constructor === Object && Object.getPrototypeOf(obj) === Object.prototype) || + constructor === undefined /* Object.create(null) */ + ) { + return hooks.dictHook ? hooks.dictHook(obj) : toPyDict(obj, hooks); + } else if (constructor === Uint8Array) { + return new Sk.builtin.bytes(obj); + } else if (constructor === Set) { + return toPySet(obj, hooks); + } else if (constructor === Map) { + const ret = new Sk.builtin.dict(); + obj.forEach((val, key) => { + ret.mp$ass_subscript(toPy(key, hooks), toPy(val, hooks)); + }); + return ret; + } else if (constructor === Sk.misceval.Suspension) { + return obj; + } else { + // all objects get proxied - previously they were converted to dictionaries + // can override this behaviour with a proxy hook + return hooks.proxyHook ? hooks.proxyHook(obj) : proxy(obj); + } + } else if (hooks.unhandledHook) { + // there aren't very many types left + // could be a symbol (unlikely) + return hooks.unhandledHook(obj); } + Sk.asserts.fail("unhandled remap case of type " + type); +} - if (obj instanceof Sk.misceval.Suspension) { +/** + * + * @param {*} obj + * @param {*} hooks + * + * This will handle any object and conver it to javascript + * + * simple objects - str, int, float, bool, tuple, list, set, dict, bytes + * are converted as you might expect + * + * str - string + * int - number or bigint (depending on the size) + * float - number + * bool - boolean + * tuple/list - array + * set - Set + * dict - object literal + * bytes - Uint8Array + * + * dict - all keys are allowed - this may cause unexpected bevaiour for non str keys + * {None: 'a', (1,): 'b', True: 'c', A(): 'd'} => {null: 'a', '1,': 'b', true: 'c', "<'A' object>": 'd'} + * and on conversion back will convert all the keys to str objects + * + * All js objects passed to this function will be returned + * + * All other python objects are wrapped + * wrapped objects have a truthy $isPyWrapped property and an unwrap method + * (used to convert back toPy) + * + * can override behaviours with hooks + * + * hooks.dictHook(pydict) - override the default behaviour from dict to object literal + * hooks.setHook(pyset) - override the default behaviour from set to Set + * hooks.unhandledHook(pyobj) - python objects that arent simple (str, None, bool, list, set, tuple, int, float) will return undefined - override this behaveiour here + * + * hooks.arrayHook(arr, pyobj) - override the array behaviour resulting from a tuple or list (get the internal array of python objects as the first argument) + * hooks.numberHook(num, pyobj) - override the number return for float, int + * hooks.bigintHoot(bigint, pyobj) - override the return of a bigint for large python integers (this might be polyfilled in older browsers) + * hooks.objectHook(obj, pyobj) - override the behaviour of a javascript object (of type object) that is about to be returned + * hooks.funcHook(func, pyobj) - override the behvaiour of javascript function that is about to be returned + */ +function toJs(obj, hooks) { + if (obj === undefined || obj === null) { return obj; } + const val = obj.valueOf(); + // for str/bool/int/float/tuple/list this returns the obvious: this.v; + // can override valueOf for skulpt objects that you want to send back and forth between python/js + if (val === null) { + return val; + } - if (Object.prototype.toString.call(obj) === "[object Array]") { - arr = []; - for (i = 0; i < obj.length; ++i) { - arr.push(Sk.ffi.remapToPy(obj[i])); + const type = typeof val; + hooks = hooks || {}; + + if (type === "string") { + return hooks.stringHook ? hooks.stringHook(val) : val; + } else if (type === "boolean") { + return val; + } else if (type === "number") { + return hooks.numberHook ? hooks.numberHook(val, obj) : val; + // pass the number and the original obj (float or int (or number)) + } else if (JSBI.__isBigInt(val)) { + // either it's a native bigint or polyfilled as an array like object + // pass the bigint (or polyfilled bigint) and the original obj (int) to the hook function + // or return the the bigint + return hooks.bigintHook ? hooks.bigintHook(val, obj) : val; + } else if (Array.isArray(val)) { + return hooks.arrayHook ? hooks.arrayHook(val, obj) : val.map((x) => toJs(x, hooks)); + // pass the array and the original obj (tuple or list (or Array)) + } else if (val.sk$object) { + // python objects are either of type object or function + // so check if they're python objects now + // these python object didn't override valueOf() + if (obj instanceof Sk.builtin.dict) { + return hooks.dictHook ? hooks.dictHook(obj) : toJsHashMap(obj, hooks); + } else if (obj instanceof Sk.builtin.set) { + return hooks.setHook ? hooks.setHook(obj) : new Set(toJsArray(obj, hooks)); + } else { + // a wrap protocol would set $isPyWrapped = true, and an unwrap function to be called in toPy + return hooks.unhandledHook ? hooks.unhandledHook(obj) : undefined; } - return new Sk.builtin.list(arr); + // fall through to unhandled hook - or return undefined + } else if (type === "object") { + return hooks.objectHook ? hooks.objectHook(val, obj) : val; + // might be a Uint8Array or some other js object that was proxied + // pass this val, obj to the objectHook if defined + // if no hook function just return the val which is not a python object + } else if (type === "function") { + // likely the result of a proxied function + // if no hook function just return the val which is not a python object + return hooks.funcHook ? hooks.funcHook(val, obj) : val; } - if (typeof obj === "object") { - kvs = []; - for (k in obj) { - kvs.push(Sk.ffi.remapToPy(k)); - kvs.push(Sk.ffi.remapToPy(obj[k])); + // we really shouldn't get here - what's left - type symbol? + Sk.asserts.fail("unhandled type " + type); +} + +/** @returns a bool based on whether it is python truthy or not. Can also hand js values */ +function isTrue(obj) { + // basically the logic for Sk.misceval.isTrue - here for convenience + return obj != null && obj.nb$bool ? obj.nb$bool() : obj.sq$length ? obj.sq$length() !== 0 : Boolean(obj); +} + +function toJsNumber(obj) { + return Number(obj); +} +function toJsString(obj) { + return String(obj); +} + +function toJsArray(obj, hooks) { + return Array.from(obj, (x) => toJs(x, hooks)); +} + +function toJsHashMap(dict, hooks) { + const obj = {}; + dict.$items().forEach(([key, val]) => { + // if non str keys are sent to this function it may behave unexpectedly (but it won't fail) + obj[key.valueOf()] = toJs(val, hooks); + }); + return obj; +} + +function numberToPy(val) { + if (Number.isInteger(val)) { + if (Math.abs(val) < Number.MAX_SAFE_INTEGER) { + return new Sk.builtin.int_(val); } - return new Sk.builtin.dict(kvs); + return new Sk.builtin.int_(JSBI.BigInt(val)); } + return new Sk.builtin.float_(val); +} - if (typeof obj === "string") { - return new Sk.builtin.str(obj); - } +const isInteger = /^-?\d+$/; - if (typeof obj === "number") { - return Sk.builtin.assk$(obj); +function toPyNumber(obj) { + const type = typeof obj; + if (type === "number") { + return numberToPy(obj); } - - if (typeof obj === "boolean") { - return new Sk.builtin.bool(obj); - } else if (typeof obj === "undefined") { - return Sk.builtin.none.none$; + if (type === "string") { + if (obj.match(isInteger)) { + return new Sk.builtin.int_(obj); + } + return new Sk.builtin.float_(parseFloat(obj)); } - - if (typeof obj === "function") { - return new Sk.builtin.func(obj); + if (JSBI.__isBigInt(obj)) { + // either type is bigint or using the bigint polyfill + return new Sk.builtin.int_(JSBI.numberIfSafe(obj)); } + return new Sk.builtin.float_(Number(obj)); +} - Sk.asserts.fail("unhandled remap type " + typeof(obj)); -}; -Sk.exportSymbol("Sk.ffi.remapToPy", Sk.ffi.remapToPy); +function toPyFloat(num) { + return new Sk.builtin.float_(Number(num)); +} -/** - * Maps from Python dict/list/str/number to Javascript Object/Array/string/number. - * - * If obj is a - * - * @param obj {Object} Any Python object (except a function) - * - */ -Sk.ffi.remapToJs = function (obj) { - var i; - var kAsJs; - var ret; - if (obj instanceof Sk.builtin.dict) { - ret = {}; - obj.$items().forEach(([key, val]) => { - kAsJs = Sk.ffi.remapToJs(key); - // todo; assert that this is a reasonble lhs? - ret[kAsJs] = Sk.ffi.remapToJs(val); - }); - return ret; - } else if (obj instanceof Sk.builtin.list || obj instanceof Sk.builtin.tuple) { - ret = []; - for (i = 0; i < obj.v.length; ++i) { - ret.push(Sk.ffi.remapToJs(obj.v[i])); - } - return ret; - } else if (obj instanceof Sk.builtin.bool) { - return obj.v ? true : false; - } else if (obj instanceof Sk.builtin.int_) { - return Sk.builtin.asnum$(obj); - } else if (obj instanceof Sk.builtin.float_) { - return Sk.builtin.asnum$(obj); - } else if (obj instanceof Sk.builtin.lng) { - return Sk.builtin.asnum$(obj); - } else if (typeof obj === "number" || typeof obj === "boolean" || typeof obj === "string") { - return obj; - } else if (obj === undefined) { - return undefined; +function toPyStr(obj) { + return new Sk.builtin.str(obj); +} + +function toPyList(obj, hooks) { + return new Sk.builtin.list(Array.from(obj, (x) => toPy(x, hooks))); +} + +function toPySet(obj, hooks) { + return new Sk.builtin.set(Array.from(obj, (x) => toPy(x, hooks))); +} + +function toPyTuple(obj, hooks) { + return new Sk.builtin.tuple(Array.from(obj, (x) => toPy(x, hooks))); +} + +function toPyInt(num) { + if (typeof num === "number") { + num = Math.trunc(num); + return Math.abs(num) < Number.MAX_SAFE_INTEGER + ? new Sk.builtin.int_(num) + : new Sk.builtin.int_(JSBI.BigInt(num)); + } else if (JSBI.__isBigInt(num)) { + return new Sk.builtin.int_(JSBI.numberIfSafe(num)); + } else if (typeof num === "string" && num.match(isInteger)) { + return new Sk.builtin.int_(num); } else { - return obj.v; + throw new TypeError("bad type passed to toPyInt() got " + num); } -}; -Sk.exportSymbol("Sk.ffi.remapToJs", Sk.ffi.remapToJs); +} -Sk.ffi.callback = function (fn) { - if (fn === undefined) { - return fn; - } - return function () { - return Sk.misceval.apply(fn, undefined, undefined, undefined, Array.prototype.slice.call(arguments, 0)); - }; -}; -Sk.exportSymbol("Sk.ffi.callback", Sk.ffi.callback); +function toPyDict(obj, hooks) { + const ret = new Sk.builtin.dict(); + Object.entries(obj).forEach(([key, val]) => { + ret.mp$ass_subscript(new Sk.builtin.str(key), toPy(val, hooks)); + }); + return ret; +} -Sk.ffi.stdwrap = function (type, towrap) { - var inst = new type(); - inst["v"] = towrap; - return inst; -}; -Sk.exportSymbol("Sk.ffi.stdwrap", Sk.ffi.stdwrap); +// cache the proxied objects in a weakmap +const _proxied = new WeakMap(); -/** - * for when the return type might be one of a variety of basic types. - * number|string, etc. - */ -Sk.ffi.basicwrap = function (obj) { - if (obj instanceof Sk.builtin.int_) { - return Sk.builtin.asnum$(obj); - } - if (obj instanceof Sk.builtin.float_) { - return Sk.builtin.asnum$(obj); - } - if (obj instanceof Sk.builtin.lng) { - return Sk.builtin.asnum$(obj); +// use proxy if you want to proxy an arbirtrary js object +// the only flags currently used is {bound: some_js_object} +function proxy(obj, flags) { + if (obj === null || obj === undefined) { + return Sk.builtin.none.none$; } - if (typeof obj === "number" || typeof obj === "boolean") { - return obj; + const type = typeof obj; + if (type !== "object" && type !== "function") { + return toPy(obj); // don't proxy strings, numbers, bigints } - if (typeof obj === "string") { - return new Sk.builtin.str(obj); + flags = flags || {}; + const cached = _proxied.get(obj); + if (cached) { + if (flags.bound === cached.$bound) { + return cached; + } + if (!flags.name) { + flags.name = cached.$name; + } } - Sk.asserts.fail("unexpected type for basicwrap"); -}; -Sk.exportSymbol("Sk.ffi.basicwrap", Sk.ffi.basicwrap); + const ret = new JsProxy(obj, flags); + _proxied.set(obj, ret); + return ret; +} -Sk.ffi.unwrapo = function (obj) { - if (obj === undefined) { - return undefined; - } - return obj["v"]; +const pyHooks = { dictHook: (obj) => proxy(obj), unhandledHook: (obj) => String(obj) }; +// unhandled is likely only Symbols and get a string rather than undefined +const boundHook = (bound, name) => ({ + dictHook: (obj) => proxy(obj), + funcHook: (obj) => proxy(obj, { bound, name }), + unhandledHook: (obj) => String(obj), +}); +const jsHooks = { + unhandledHook: (obj) => { + const _cached = _proxied.get(obj); + if (_cached) { + return _cached; + } else if (obj.tp$call) { + const wrapped = (...args) => { + const ret = Sk.misceval.chain(obj.tp$call(args.map((x) => toPy(x, pyHooks))), (res) => + toJs(res, jsHooks) + ); + if (ret instanceof Sk.misceval.Suspension) { + // better to return a promise here then hope the javascript library will handle a suspension + return Sk.misceval.asyncToPromise(() => ret); + } + return ret; + }; + wrapped.v = obj; + wrapped.unwrap = () => obj; + wrapped.$isPyWrapped = true; + _proxied.set(obj, wrapped); + return wrapped; + } + const ret = { v: obj, $isPyWrapped: true, unwrap: () => obj }; + _proxied.set(obj, ret); + return ret; + }, }; -Sk.exportSymbol("Sk.ffi.unwrapo", Sk.ffi.unwrapo); +// we customize the dictHook and the funcHook here - we want to keep object literals as proxied objects when remapping to Py +// and we want funcs to be proxied -Sk.ffi.unwrapn = function (obj) { - if (obj === null) { - return null; +const JsProxy = Sk.abstr.buildNativeClass("Proxy", { + constructor: function JsProxy(obj, flags) { + if (obj === undefined) { + throw new Sk.builtin.TypeError("Proxy cannot be called from python"); + } + this.js$wrapped = obj; + this.$module = null; + this.$methods = Object.create(null); + this.in$repr = false; + + flags || (flags = {}); + + // make slot functions lazy + Object.defineProperties(this, this.memoized$slots); + + // determine the type and name of this proxy + if (typeof obj === "function") { + this.is$callable = true; + this.$bound = flags.bound; + this.$name = flags.name || obj.name || "(native JS)"; + if (this.$name.length <= 2) { + this.$name = this.$name + " (native JS)"; // better this than a single letter minified name + } + } else { + this.is$callable = false; + delete this.is$type; // set in memoized slots for lazy loading; + this.is$type = false; + this.$name = flags.name; + } + }, + slots: { + tp$doc: "proxy for a javascript object", + tp$hash() { + return Sk.builtin.object.prototype.tp$hash.call(this.js$wrapped); + }, + tp$getattr(pyName) { + return this.$lookup(pyName) || Sk.generic.getAttr.call(this, pyName); + }, + tp$setattr(pyName, value) { + const jsName = pyName.toString(); + if (value === undefined) { + delete this.js$wrapped[jsName]; + } else { + this.js$wrapped[jsName] = toJs(value, jsHooks); + } + }, + $r() { + if (this.is$callable) { + if (this.is$type || !this.$bound) { + return new Sk.builtin.str("<" + this.tp$name + " '" + this.$name + "'>"); + } + const boundRepr = Sk.misceval.objectRepr(proxy(this.$bound)); + return new Sk.builtin.str(""); + } else if (this.js$proto === Object.prototype) { + if (this.in$repr) { + return new Sk.builtin.str("{...}"); + } + this.in$repr = true; + const entries = Object.entries(this.js$wrapped).map(([key, val]) => { + val = toPy(val, boundHook(this.js$wrapped, key)); + return "'" + key + "': " + Sk.misceval.objectRepr(val); + }); + const ret = new Sk.builtin.str("proxyobject({" + entries.join(", ") + "})"); + this.in$repr = false; + return ret; + } + const object = this.tp$name === "proxyobject" ? "object" : "proxyobject"; + return new Sk.builtin.str("<" + this.tp$name + " " + object + ">"); + }, + tp$as_sequence_or_mapping: true, + mp$subscript(pyItem) { + // todo should we account for -1 i.e. array like subscripts + const ret = this.$lookup(pyItem); + if (ret === undefined) { + throw new Sk.builtin.LookupError(pyItem); + } + return ret; + }, + mp$ass_subscript(pyItem, value) { + return this.tp$setattr(pyItem, value); + }, + sq$contains(item) { + return toJs(item) in this.js$wrapped; + }, + ob$eq(other) { + return this.js$wrapped === other.js$wrapped; + }, + ob$ne(other) { + return this.js$wrapped !== other.js$wrapped; + }, + tp$as_number: true, + nb$bool() { + // we could just check .constructor but some libraries delete it! + if (this.js$proto === Object.prototype) { + return Object.keys(this.js$wrapped).length > 0; + } else if (this.sq$length) { + return this.sq$length() > 0; + } else { + return true; + } + }, + }, + methods: { + __dir__: { + $meth() { + const object_dir = Sk.misceval.callsimArray(Sk.builtin.object.prototype.__dir__, [this]).valueOf(); + return new Sk.builtin.list(object_dir.concat(Array.from(this.$dir, (x) => new Sk.builtin.str(x)))); + }, + $flags: { NoArgs: true }, + }, + __new__: { + // this is effectively a static method + $meth(js_proxy, ...args) { + if (!(js_proxy instanceof JsProxy)) { + throw new Sk.builtin.TypeError( + "expected a proxy object as the first argument not " + Sk.abstr.typeName(js_proxy) + ); + } + try { + // let javascript throw errors if it wants + return js_proxy.$new(args); + } catch (e) { + if (e instanceof TypeError && e.message.includes("not a constructor")) { + throw new Sk.builtin.TypeError(Sk.misceval.objectRepr(js_proxy) + " is not a constructor"); + } + throw e; + } + }, + $flags: { MinArgs: 1 }, + }, + __call__: { + $meth(args, kwargs) { + if (typeof this.js$wrapped !== "function") { + throw new Sk.builtin.TypeError("'" + this.tp$name + "' object is not callable"); + } + return this.$call(args, kwargs); + }, + $flags: { FastCall: true }, + }, + keys: { + $meth() { + return new Sk.builtin.list(Object.keys(this.js$wrapped).map((x) => new Sk.builtin.str(x))); + }, + $flags: { NoArgs: true }, + }, + get: { + $meth(pyName, _default) { + return this.$lookup(pyName) || _default || Sk.builtin.none.none$; + }, + $flags: { MinArgs: 1, MaxArgs: 2 }, + }, + }, + getsets: { + __class__: { + $get() { + return toPy(this.js$wrapped.constructor, pyHooks); + }, + $set() { + throw new Sk.builtin.TypeError("not writable"); + }, + }, + __name__: { + $get() { + return new Sk.builtin.str(this.$name); + }, + }, + __module__: { + $get() { + return this.$module || Sk.builtin.none.none$; + }, + $set(v) { + this.$module = v; + }, + }, + }, + proto: { + valueOf() { + return this.js$wrapped; + }, + $new(args, kwargs) { + Sk.abstr.checkNoKwargs("__new__", kwargs); + return toPy(new this.js$wrapped(...args.map((x) => toJs(x, jsHooks))), { + dictHook: (obj) => proxy(obj), + proxyHook: (obj) => proxy(obj, { name: this.$name }), + }); + }, + $call(args, kwargs) { + Sk.abstr.checkNoKwargs("__call__", kwargs); + return Sk.misceval.chain( + this.js$wrapped.apply( + this.$bound, + args.map((x) => toJs(x, jsHooks)) + ), + (res) => (res instanceof Promise ? Sk.misceval.promiseToSuspension(res) : res), + (res) => toPy(res, pyHooks) + ); + }, + $lookup(pyName) { + const jsName = pyName.toString(); + const attr = this.js$wrapped[jsName]; + if (attr !== undefined) { + // here we override the funcHook to pass the bound object + return toPy(attr, boundHook(this.js$wrapped, jsName)); + } else if (jsName in this.js$wrapped) { + // do we actually have this property? + return Sk.builtin.none.none$; + } + }, + // only get these if we need them + memoized$slots: { + js$proto: { + configurable: true, + get() { + delete this.js$proto; + return (this.js$proto = Object.getPrototypeOf(this.js$wrapped)); + }, + }, + $dir: { + configurable: true, + get() { + const dir = new Set(); + // loop over enumerable properties + for (let prop in this.js$wrapped) { + dir.add(prop); + } + return dir; + }, + }, + tp$iter: { + configurable: true, + get() { + delete this.tp$iter; + if (this.js$wrapped[Symbol.iterator] !== undefined) { + return (this.tp$iter = () => { + return proxy(this.js$wrapped[Symbol.iterator]()); + }); + } + }, + }, + tp$iternext: { + configurable: true, + get() { + delete this.tp$iternext; + if (this.js$wrapped.next !== undefined) { + return (this.tp$iternext = () => { + const nxt = this.js$wrapped.next().value; + return nxt && toPy(nxt, pyHooks); + }); + } + }, + }, + sq$length: { + configurable: true, + get() { + delete this.sq$length; + if (!this.is$callable && this.js$wrapped.length !== undefined) { + return (this.sq$length = () => this.js$wrapped.length); + } + }, + }, + tp$call: { + configurable: true, + get() { + delete this.tp$call; + if (this.is$callable) { + return (this.tp$call = this.is$type ? this.$new : this.$call); + } + }, + }, + tp$name: { + configurable: true, + get() { + delete this.tp$name; + if (!this.is$callable) { + const obj = this.js$wrapped; + let tp$name = + obj[Symbol.toStringTag] || + this.$name || + (obj.constructor && obj.constructor.name) || + "proxyobject"; + if (tp$name === "Object") { + tp$name = this[Symbol.toStringTag]; + tp$name = "proxyobject"; + } else if (tp$name.length <= 2) { + // we might have a better name in the cache so check there... + tp$name = proxy(obj.constructor).$name; + } + return (this.tp$name = tp$name); + } else { + return (this.tp$name = this.is$type ? "proxyclass" : this.$bound ? "proxymethod" : "proxyfunction"); + } + }, + }, + is$type: { + configurable: true, + get() { + delete this.is$type; + // we already know if we're a function + const jsFunc = this.js$wrapped; + const proto = jsFunc.prototype; + if (proto === undefined) { + // Arrow functions and shorthand methods don't get a prototype + // neither do native js functions like requestAnimationFrame, JSON.parse + // Proxy doesn't get a prototype but must be called with new - it's the only one I know + // How you'd use Proxy in python I have no idea + return (this.is$type = jsFunc === window.Proxy); + } + const maybeConstructor = checkBodyIsMaybeConstructor(jsFunc); + if (maybeConstructor === true) { + // definitely a constructor and needs new + return (this.is$type = true); + } else if (maybeConstructor === false) { + // Number, Symbol, Boolean, BigInt, String + return (this.is$type = false); + } + const protoLen = Object.getOwnPropertyNames(proto).length; + if (protoLen > 1) { + // if the function object has a prototype with more than just constructor, intention is to be used as a constructor + return (this.is$type = true); + } + return (this.is$type = Object.getPrototypeOf(proto) !== Object.prototype); + // we could be a subclass with only constructor on the prototype + // if our prototype's __proto__ is Object.prototype then we are the most base function + // the most likely option is that `this` should be whatever `this.$bound` is, rather than using new + // example x is this.$bound and shouldn't be called with new + // var x = {foo: function() {this.bar='foo'}} + // Sk.misceval.Break is a counter example + // better to fail with Sk.misceval.Break() (which may have a type guard) than fail by calling new x.foo() + }, + }, + }, + }, + flags: { + sk$acceptable_as_base_class: false, + }, +}); + +const is_constructor = /^class|^function[a-zA-Z\d\(\)\{\s]+\[native code\]\s+\}$/; + +const getFunctionBody = Function.prototype.toString; +const noNewNeeded = new Set([Number, String, Symbol, Boolean]); +// Some js builtins that shouldn't be called with new +// these are unlikely to be proxied by the user but you never know +if (typeof Sk.global.BigInt !== "undefined") { + noNewNeeded.add(Sk.global.BigInt); +} + +function checkBodyIsMaybeConstructor(obj) { + const body = getFunctionBody.call(obj); + const match = body.match(is_constructor); + if (match === null) { + return null; // Not a constructor + } else if (match[0] === "class") { + return true; + } else { + // some native constructors shouldn't have new + return !noNewNeeded.has(obj); } - return obj["v"]; -}; -Sk.exportSymbol("Sk.ffi.unwrapn", Sk.ffi.unwrapn); +} diff --git a/src/lib/re.js b/src/lib/re.js index 8e901b7162..ae2b1f0558 100644 --- a/src/lib/re.js +++ b/src/lib/re.js @@ -77,8 +77,8 @@ var $builtinmodule = function (name) { } maxsplit = Sk.builtin.asnum$(maxsplit); - pat = Sk.ffi.unwrapo(pattern); - str = Sk.ffi.unwrapo(string); + pat = Sk.ffi.remapToJs(pattern); + str = Sk.ffi.remapToJs(string); // Convert pat from Python to Javascript regex syntax pat = convert(pat); @@ -142,8 +142,8 @@ var $builtinmodule = function (name) { throw new Sk.builtin.TypeError("flags must be a number"); } - pat = Sk.ffi.unwrapo(pattern); - str = Sk.ffi.unwrapo(string); + pat = Sk.ffi.remapToJs(pattern); + str = Sk.ffi.remapToJs(string); // Convert pat from Python to Javascript regex syntax pat = convert(pat); From 892621e046bccc3bafd295e2b22ffe8440167d67 Mon Sep 17 00:00:00 2001 From: mrcork Date: Thu, 18 Feb 2021 14:03:17 +0800 Subject: [PATCH 088/137] adjusted document api make it a lazy module and return proxy objects --- src/abstract.js | 5 +- src/lib/document.js | 186 +++++++------------------------------------- 2 files changed, 29 insertions(+), 162 deletions(-) diff --git a/src/abstract.js b/src/abstract.js index 176a0924c8..2e679e2daa 100644 --- a/src/abstract.js +++ b/src/abstract.js @@ -1414,11 +1414,12 @@ Sk.abstr.buildIteratorClass = function (typename, iterator) { Sk.abstr.built$iterators = []; -Sk.abstr.setUpModuleMethods = function (module_name, module, method_defs) { +Sk.abstr.setUpModuleMethods = function (module_name, mod, method_defs) { Object.entries(method_defs).forEach(([method_name, method_def]) => { method_def.$name = method_def.$name || method_name; // operator e.g. some methods share method_defs - module[method_name] = new Sk.builtin.sk_method(method_def, null, module_name); + mod[method_name] = new Sk.builtin.sk_method(method_def, null, module_name); }); + return mod; }; /** diff --git a/src/lib/document.js b/src/lib/document.js index 440089d962..401514cc95 100644 --- a/src/lib/document.js +++ b/src/lib/document.js @@ -1,162 +1,28 @@ -var $builtinmodule = function (name) { - var elementClass; - var mod = {__name__: new Sk.builtin.str("document")}; - - mod.getElementById = new Sk.builtin.func(function (id) { - var result = document.getElementById(id.v); - if (result) { - return Sk.misceval.callsimArray(mod.Element, [result]); - } - return Sk.builtin.none.none$; - }); - - mod.createElement = new Sk.builtin.func(function (eName) { - var r = document.createElement(eName.v); - if (r) { - return Sk.misceval.callsimArray(mod.Element, [r]); - } - }); - - - mod.getElementsByTagName = new Sk.builtin.func(function (tag) { - var r = document.getElementsByTagName(tag.v) - var reslist = []; - for (var i = r.length - 1; i >= 0; i--) { - reslist.push(Sk.misceval.callsimArray(mod.Element, [r[i]])) - } - return new Sk.builtin.list(reslist) - }); - - mod.getElementsByClassName = new Sk.builtin.func(function (cname) { - var r = document.getElementsByClassName(cname.v); - var reslist = []; - for (var i = 0; i < r.length; i++) { - reslist.push(Sk.misceval.callsimArray(mod.Element, [r[i]])); - } - ; - return new Sk.builtin.list(reslist); - }); - - mod.getElementsByName = new Sk.builtin.func(function (cname) { - var r = document.getElementsByName(cname.v); - var reslist = []; - for (var i = 0; i < r.length; i++) { - reslist.push(Sk.misceval.callsimArray(mod.Element, [r[i]])); - } - ; - return new Sk.builtin.list(reslist); - }); - - mod.currentDiv = new Sk.builtin.func(function () { - if (Sk.divid !== undefined) { - return new Sk.builtin.str(Sk.divid) - } - else { - throw new Sk.builtin.AttributeError("There is no value set for divid"); - } - }) - - elementClass = function ($gbl, $loc) { - /* - Notes: self['$d'] is the dictionary used by the generic.getAttr mechanism for an object. - for various reasons if you create a class in Javascript and have self.xxxx instance - variables, you cannot say instance.xxx and get the value of the instance variable unless - it is stored in the self['$d'] object. This seems like a duplication of storage to me - but that is how it works right now (5/2013) - - Writing your own __getattr__ is also an option but this gets very tricky when an attr is - a method... - */ - $loc.__init__ = new Sk.builtin.func(function (self, elem) { - self.v = elem - self.innerHTML = elem.innerHTML - self.innerText = elem.innerText - if (elem.value !== undefined) { - self.value = elem.value - Sk.abstr.objectSetItem(self['$d'], new Sk.builtin.str('value'), new Sk.builtin.str(self.value)) - } - - if (elem.checked !== undefined) { - self.checked = elem.checked - Sk.abstr.objectSetItem(self['$d'], new Sk.builtin.str('checked'), new Sk.builtin.str(self.checked)) - } - - Sk.abstr.objectSetItem(self['$d'], new Sk.builtin.str('innerHTML'), new Sk.builtin.str(self.innerHTML)) - Sk.abstr.objectSetItem(self['$d'], new Sk.builtin.str('innerText'), new Sk.builtin.str(self.innerText)) - - }) - - $loc.__setattr__ = new Sk.builtin.func(function (self, key, value) { - key = Sk.ffi.remapToJs(key); - if (key === 'innerHTML') { - self.innerHTML = value - self.v.innerHTML = value.v - Sk.abstr.objectSetItem(self['$d'], new Sk.builtin.str('innerHTML'), value) - } - if (key === 'innerText') { - self.innerText = value - self.v.innerText = value.v - Sk.abstr.objectSetItem(self['$d'], new Sk.builtin.str('innerText'), value) - } - }); - - - $loc.appendChild = new Sk.builtin.func(function (self, ch) { - self.v.appendChild(ch.v); - }); - - $loc.removeChild = new Sk.builtin.func(function (self, node) { - self.v.removeChild(node.v) - }) - - // getCSS - - $loc.getCSS = new Sk.builtin.func(function (self, key) { - return new Sk.builtin.str(self.v.style[key.v]); - }); - - - $loc.setCSS = new Sk.builtin.func(function (self, attr, value) { - self.v.style[attr.v] = value.v - - }) - - $loc.getAttribute = new Sk.builtin.func(function (self, key) { - var res = self.v.getAttribute(key.v) - if (res) { - return new Sk.builtin.str(res) - } else { - return Sk.builtin.none.none$; - } - }); - - $loc.setAttribute = new Sk.builtin.func(function (self, attr, value) { - self.v.setAttribute(attr.v, value.v) - }); - - $loc.getProperty = new Sk.builtin.func(function (self, key) { - var res = self.v[key.v] - if (res) { - return new Sk.builtin.str(res) - } else { - return Sk.builtin.none.none$; - } - }); - - $loc.__str__ = new Sk.builtin.func(function (self) { - console.log(self.v.tagName); - return new Sk.builtin.str(self.v.tagName) - }) - - $loc.__repr__ = new Sk.builtin.func(function (self) { - return new Sk.builtin.str('[DOM Element]') - }) - - +function $builtinmodule() { + var documentMod = { + __name__: new Sk.builtin.str("document"), }; - - mod.Element = Sk.misceval.buildClass(mod, elementClass, 'Element', []); - - return mod; - + var jsDocument = Sk.global.document; + var documentProxyObject = Sk.ffi.toPy(jsDocument); + + Sk.abstr.setUpModuleMethods("document", documentMod, { + __getattr__: { + $meth(pyName) { + var ret = documentProxyObject.tp$getattr(pyName); + if (ret === undefined) { + throw new Sk.builtin.AttributeError(pyName.toString()); + } + return ret; + }, + $flags: { OneArg: true }, + }, + __dir__: { + $meth() { + // may want to include more than this + return Sk.ffi.toPy(Object.keys(document)); + }, + $flags: { NoArgs: true }, + }, + }); + return documentMod; } From 9fde1f2c9253ba1dc5a61bca88cb78813a785c3f Mon Sep 17 00:00:00 2001 From: mrcork Date: Thu, 18 Feb 2021 14:04:42 +0800 Subject: [PATCH 089/137] p5 example implementation --- src/lib/p5.js | 83 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 src/lib/p5.js diff --git a/src/lib/p5.js b/src/lib/p5.js new file mode 100644 index 0000000000..327730146e --- /dev/null +++ b/src/lib/p5.js @@ -0,0 +1,83 @@ +function $builtinmodule() { + + const { + builtins: __builtins__, + builtin: { + str: pyStr, + func: pyFunc, + none: { none$: pyNone }, + }, + misceval: { callsimArray: pyCall }, + ffi: { toPy }, + } = Sk; + + + const mod = { + __name__: new pyStr("p5"), + p5: toPy(window.p5), + __doc__: new pyStr("A skulpt implementation of the p5 library"), + }; + + // override _start to a plain function and call this in run when we need it + let _start; + Object.defineProperty(window.p5.prototype, "_start", { + get() { + return () => {}; + }, + set(val) { + _start = val; + }, + configurable: true, + }); + + function sketch(p) { + for (let i in window.p5.prototype) { + const asStr = new pyStr(i); + const mangled = asStr.$mangled; + // it would be crazy to override builtins like print + if (!(mangled in __builtins__)) { + mod[mangled] = p.tp$getattr(asStr); + } + } + }; + + // create an instance of p5 and assign all the attributes to mod + const p = pyCall(mod.p5, [new pyFunc(sketch), toPy(Sk.canvas)]); + + const wrapFunc = (func) => () => { + try { + pyCall(func); + } catch (e) { + Sk.uncaughtException && Sk.uncaughtException(e); + } + // note we can't suspend because promises are just ignored in these methods + }; + + mod.run = new pyFunc(function run() { + const main = Sk.sysmodules.quick$lookup(new pyStr("__main__")).$d; + delete window.p5.prototype._start; + const pInstance = p.valueOf(); + pInstance._start = _start; + + ["preload", "setup", "draw"].forEach((methodName) => { + const method = main[methodName]; + if (method !== undefined) { + pInstance[methodName] = wrapFunc(method); + } + }); + + // p5 wants to change the global namespace of things like frameCount, key. So let it + const _setProperty = pInstance._setProperty; + pInstance._setProperty = function (prop, val) { + _setProperty.call(this, prop, val); + const asStr = new pyStr(prop); + const mangled = asStr.$mangled; + mod[mangled] = main[mangled] = p.tp$getattr(asStr); + }; + + pInstance._start(); + return pyNone; + }); + + return mod; +} From e4aa67ef5f78cc4ce5f441485f780a142ee1a069 Mon Sep 17 00:00:00 2001 From: Stu Cork Date: Fri, 16 Jul 2021 12:14:37 +0000 Subject: [PATCH 090/137] p5 example implementation --- src/lib/p5.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/lib/p5.js b/src/lib/p5.js index 327730146e..c6678bd8d5 100644 --- a/src/lib/p5.js +++ b/src/lib/p5.js @@ -1,5 +1,4 @@ function $builtinmodule() { - const { builtins: __builtins__, builtin: { @@ -11,7 +10,6 @@ function $builtinmodule() { ffi: { toPy }, } = Sk; - const mod = { __name__: new pyStr("p5"), p5: toPy(window.p5), @@ -23,7 +21,7 @@ function $builtinmodule() { Object.defineProperty(window.p5.prototype, "_start", { get() { return () => {}; - }, + }, set(val) { _start = val; }, @@ -39,7 +37,7 @@ function $builtinmodule() { mod[mangled] = p.tp$getattr(asStr); } } - }; + } // create an instance of p5 and assign all the attributes to mod const p = pyCall(mod.p5, [new pyFunc(sketch), toPy(Sk.canvas)]); From 80b99e9937fafdf7a79eb1bfdd2ae6abdd799204 Mon Sep 17 00:00:00 2001 From: mrcork Date: Thu, 18 Feb 2021 14:44:55 +0800 Subject: [PATCH 091/137] include a js module with access to the window object --- src/lib/js.js | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 src/lib/js.js diff --git a/src/lib/js.js b/src/lib/js.js new file mode 100644 index 0000000000..416802f2c1 --- /dev/null +++ b/src/lib/js.js @@ -0,0 +1,6 @@ +function $builtinmodule() { + const mod = { __name__: new Sk.builtin.str("js") }; + mod.window = Sk.ffi.proxy(window); + Sk.abstr.objectSetItem(Sk.sysmodules, new Sk.builtin.str("js.window"), mod.window); + return mod; +} From 7ac7fcc0641d6516296bf1880b3de60d984120db Mon Sep 17 00:00:00 2001 From: Nick McIntyre Date: Wed, 17 Mar 2021 09:05:58 -0500 Subject: [PATCH 092/137] Add p5 callbacks and examples (#17) * Add p5 callbacks --- src/lib/p5.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lib/p5.js b/src/lib/p5.js index c6678bd8d5..0b1ac2dfaf 100644 --- a/src/lib/p5.js +++ b/src/lib/p5.js @@ -57,7 +57,10 @@ function $builtinmodule() { const pInstance = p.valueOf(); pInstance._start = _start; - ["preload", "setup", "draw"].forEach((methodName) => { + ["preload", "setup", "draw", "deviceMoved", "deviceTurned", "deviceShaken", + "windowResized", "keyPressed", "keyReleased", "keyTyped", "mousePressed", + "mouseReleased", "mouseClicked", "doubleClicked", "mouseMoved", "mouseDragged", + "mouseWheel", "touchStarted", "touchMoved", "touchEnded"].forEach((methodName) => { const method = main[methodName]; if (method !== undefined) { pInstance[methodName] = wrapFunc(method); From 8a046430428cf2397e50445a91ee06b19efd5579 Mon Sep 17 00:00:00 2001 From: Stu Cork Date: Fri, 16 Jul 2021 12:15:37 +0000 Subject: [PATCH 093/137] Add a kill method to p5 implemenation --- src/lib/p5.js | 40 ++++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/src/lib/p5.js b/src/lib/p5.js index 0b1ac2dfaf..a3e8dd712a 100644 --- a/src/lib/p5.js +++ b/src/lib/p5.js @@ -10,6 +10,9 @@ function $builtinmodule() { ffi: { toPy }, } = Sk; + // setup a p5 object on Sk if not already there + Sk.p5 || (Sk.p5 = {}); + const mod = { __name__: new pyStr("p5"), p5: toPy(window.p5), @@ -17,6 +20,10 @@ function $builtinmodule() { }; // override _start to a plain function and call this in run when we need it + // this is kind of hacky + // _start is set in the constructor and then called + // by overriding the prototype means we can delay the call to _start + // which p5 does on initialization to get the methods in the namespace let _start; Object.defineProperty(window.p5.prototype, "_start", { get() { @@ -29,6 +36,7 @@ function $builtinmodule() { }); function sketch(p) { + // p is a python object since we used a python function for (let i in window.p5.prototype) { const asStr = new pyStr(i); const mangled = asStr.$mangled; @@ -40,7 +48,10 @@ function $builtinmodule() { } // create an instance of p5 and assign all the attributes to mod - const p = pyCall(mod.p5, [new pyFunc(sketch), toPy(Sk.canvas)]); + const p = pyCall(mod.p5, [new pyFunc(sketch), toPy(Sk.p5.node || Sk.canvas)]); + const pInstance = p.valueOf(); + Sk.p5.instance = pInstance; + Sk.p5.kill = pInstance.remove.bind(pInstance); const wrapFunc = (func) => () => { try { @@ -54,13 +65,30 @@ function $builtinmodule() { mod.run = new pyFunc(function run() { const main = Sk.sysmodules.quick$lookup(new pyStr("__main__")).$d; delete window.p5.prototype._start; - const pInstance = p.valueOf(); pInstance._start = _start; - ["preload", "setup", "draw", "deviceMoved", "deviceTurned", "deviceShaken", - "windowResized", "keyPressed", "keyReleased", "keyTyped", "mousePressed", - "mouseReleased", "mouseClicked", "doubleClicked", "mouseMoved", "mouseDragged", - "mouseWheel", "touchStarted", "touchMoved", "touchEnded"].forEach((methodName) => { + [ + "preload", + "setup", + "draw", + "deviceMoved", + "deviceTurned", + "deviceShaken", + "windowResized", + "keyPressed", + "keyReleased", + "keyTyped", + "mousePressed", + "mouseReleased", + "mouseClicked", + "doubleClicked", + "mouseMoved", + "mouseDragged", + "mouseWheel", + "touchStarted", + "touchMoved", + "touchEnded", + ].forEach((methodName) => { const method = main[methodName]; if (method !== undefined) { pInstance[methodName] = wrapFunc(method); From 819f933402ec9b340241c94fc29a634077d33ad1 Mon Sep 17 00:00:00 2001 From: stu Date: Tue, 22 Nov 2022 14:04:19 +0800 Subject: [PATCH 094/137] Updates to ffi API --- src/abstract.js | 12 ++--- src/ffi.js | 85 ++++++++++++++++++++------------- src/lib/document.js | 23 +++++---- src/lib/js.js | 6 --- src/lib/p5.js | 112 -------------------------------------------- src/object.js | 4 +- 6 files changed, 71 insertions(+), 171 deletions(-) delete mode 100644 src/lib/js.js delete mode 100644 src/lib/p5.js diff --git a/src/abstract.js b/src/abstract.js index 2e679e2daa..351a4f7137 100644 --- a/src/abstract.js +++ b/src/abstract.js @@ -1334,8 +1334,8 @@ Sk.abstr.buildNativeClass = function (typename, options) { }); - if (typeobject.prototype.hasOwnProperty("tp$iter")) { - typeobject.prototype[Symbol.iterator] = function () { + if (type_proto.hasOwnProperty("tp$iter")) { + type_proto[Symbol.iterator] = function () { return this.tp$iter()[Symbol.iterator](); }; } @@ -1401,11 +1401,9 @@ Sk.abstr.buildIteratorClass = function (typename, iterator) { ret.prototype[Symbol.iterator] = function () { return { next: () => { - const nxt = this.tp$iternext(); - if (nxt === undefined) { - return {done: true}; - } - return {value: nxt, done: false}; + const value = this.tp$iternext(); + const done = value === undefined; + return {value, done}; } }; }; diff --git a/src/ffi.js b/src/ffi.js index 428112c9a2..c1a99d0ef6 100644 --- a/src/ffi.js +++ b/src/ffi.js @@ -29,6 +29,9 @@ Sk.ffi = { proxy, }; +const OBJECT_PROTO = Object.prototype; +const FUNC_PROTO = Function.prototype; + /** * maps from Javascript Object/Array/string to Python dict/list/str. * @@ -69,10 +72,7 @@ function toPy(obj, hooks) { return new Sk.builtin.list(obj.map((x) => toPy(x, hooks))); } else if (type === "object") { const constructor = obj.constructor; // it's possible that a library deleted the constructor - if ( - (constructor === Object && Object.getPrototypeOf(obj) === Object.prototype) || - constructor === undefined /* Object.create(null) */ - ) { + if (constructor === Object && Object.getPrototypeOf(obj) === OBJECT_PROTO || constructor === undefined /* Object.create(null) */) { return hooks.dictHook ? hooks.dictHook(obj) : toPyDict(obj, hooks); } else if (constructor === Uint8Array) { return new Sk.builtin.bytes(obj); @@ -336,26 +336,35 @@ const jsHooks = { const _cached = _proxied.get(obj); if (_cached) { return _cached; - } else if (obj.tp$call) { - const wrapped = (...args) => { - const ret = Sk.misceval.chain(obj.tp$call(args.map((x) => toPy(x, pyHooks))), (res) => - toJs(res, jsHooks) - ); - if (ret instanceof Sk.misceval.Suspension) { - // better to return a promise here then hope the javascript library will handle a suspension + } + const pyWrapped = { v: obj, $isPyWrapped: true, unwrap: () => obj }; + if (obj.tp$call === undefined) { + _proxied.set(obj, pyWrapped); + return pyWrapped; + } + const pyWrappedCallable = (...args) => { + args = args.map((x) => toPy(x, pyHooks)); + let ret = Sk.misceval.tryCatch( + () => Sk.misceval.chain(obj.tp$call(args), (res) => toJs(res, jsHooks)), + (e) => { + if (Sk.uncaughtException) { + Sk.uncaughtException(e); + } else { + throw e; + } + } + ); + while (ret instanceof Sk.misceval.Suspension) { + // better to return a promise here then hope the javascript library will handle a suspension + if (!ret.optional) { return Sk.misceval.asyncToPromise(() => ret); } - return ret; - }; - wrapped.v = obj; - wrapped.unwrap = () => obj; - wrapped.$isPyWrapped = true; - _proxied.set(obj, wrapped); - return wrapped; - } - const ret = { v: obj, $isPyWrapped: true, unwrap: () => obj }; - _proxied.set(obj, ret); - return ret; + ret = ret.resume(); + } + return ret; + }; + _proxied.set(obj, Object.assign(pyWrappedCallable, pyWrapped)); + return pyWrappedCallable; }, }; // we customize the dictHook and the funcHook here - we want to keep object literals as proxied objects when remapping to Py @@ -414,7 +423,7 @@ const JsProxy = Sk.abstr.buildNativeClass("Proxy", { } const boundRepr = Sk.misceval.objectRepr(proxy(this.$bound)); return new Sk.builtin.str(""); - } else if (this.js$proto === Object.prototype) { + } else if (this.js$proto === OBJECT_PROTO) { if (this.in$repr) { return new Sk.builtin.str("{...}"); } @@ -454,7 +463,7 @@ const JsProxy = Sk.abstr.buildNativeClass("Proxy", { tp$as_number: true, nb$bool() { // we could just check .constructor but some libraries delete it! - if (this.js$proto === Object.prototype) { + if (this.js$proto === OBJECT_PROTO) { return Object.keys(this.js$wrapped).length > 0; } else if (this.sq$length) { return this.sq$length() > 0; @@ -466,8 +475,8 @@ const JsProxy = Sk.abstr.buildNativeClass("Proxy", { methods: { __dir__: { $meth() { - const object_dir = Sk.misceval.callsimArray(Sk.builtin.object.prototype.__dir__, [this]).valueOf(); - return new Sk.builtin.list(object_dir.concat(Array.from(this.$dir, (x) => new Sk.builtin.str(x)))); + const proxy_dir = Sk.misceval.callsimArray(Sk.builtin.type.prototype.__dir__, [JsProxy]).valueOf(); + return new Sk.builtin.list(proxy_dir.concat(Array.from(this.$dir, (x) => new Sk.builtin.str(x)))); }, $flags: { NoArgs: true }, }, @@ -581,12 +590,16 @@ const JsProxy = Sk.abstr.buildNativeClass("Proxy", { $dir: { configurable: true, get() { - const dir = new Set(); - // loop over enumerable properties - for (let prop in this.js$wrapped) { - dir.add(prop); + const dir = []; + // just looping over enumerable properties can hide a lot of properties + // especially in es6 classes + let obj = this.js$wrapped; + + while (obj != null && obj !== OBJECT_PROTO && obj !== FUNC_PROTO) { + dir.push(...Object.getOwnPropertyNames(obj)); + obj = Object.getPrototypeOf(obj); } - return dir; + return new Set(dir); }, }, tp$iter: { @@ -597,6 +610,12 @@ const JsProxy = Sk.abstr.buildNativeClass("Proxy", { return (this.tp$iter = () => { return proxy(this.js$wrapped[Symbol.iterator]()); }); + } else { + return (this.tp$iter = () => { + // we could set it to undefined but because we have a __getitem__ + // python tries to use seq_iter which will result in a 0 LookupError, which is confusing + throw new Sk.builtin.TypeError(Sk.misceval.objectRepr(this) + " is not iterable"); + }); } }, }, @@ -681,7 +700,7 @@ const JsProxy = Sk.abstr.buildNativeClass("Proxy", { // if the function object has a prototype with more than just constructor, intention is to be used as a constructor return (this.is$type = true); } - return (this.is$type = Object.getPrototypeOf(proto) !== Object.prototype); + return (this.is$type = Object.getPrototypeOf(proto) !== OBJECT_PROTO); // we could be a subclass with only constructor on the prototype // if our prototype's __proto__ is Object.prototype then we are the most base function // the most likely option is that `this` should be whatever `this.$bound` is, rather than using new @@ -700,7 +719,7 @@ const JsProxy = Sk.abstr.buildNativeClass("Proxy", { const is_constructor = /^class|^function[a-zA-Z\d\(\)\{\s]+\[native code\]\s+\}$/; -const getFunctionBody = Function.prototype.toString; +const getFunctionBody = FUNC_PROTO.toString; const noNewNeeded = new Set([Number, String, Symbol, Boolean]); // Some js builtins that shouldn't be called with new // these are unlikely to be proxied by the user but you never know diff --git a/src/lib/document.js b/src/lib/document.js index 401514cc95..53d2848b08 100644 --- a/src/lib/document.js +++ b/src/lib/document.js @@ -1,25 +1,24 @@ function $builtinmodule() { - var documentMod = { - __name__: new Sk.builtin.str("document"), - }; - var jsDocument = Sk.global.document; - var documentProxyObject = Sk.ffi.toPy(jsDocument); + const { + builtin: { str: pyStr }, + misceval: { callsimArray: pyCall }, + ffi: { toPy }, + abstr: { gattr }, + } = Sk; + + const documentMod = { __name__: new pyStr("document") }; + const documentProxy = toPy(Sk.global.document); Sk.abstr.setUpModuleMethods("document", documentMod, { __getattr__: { $meth(pyName) { - var ret = documentProxyObject.tp$getattr(pyName); - if (ret === undefined) { - throw new Sk.builtin.AttributeError(pyName.toString()); - } - return ret; + return gattr(documentProxy, pyName, true); }, $flags: { OneArg: true }, }, __dir__: { $meth() { - // may want to include more than this - return Sk.ffi.toPy(Object.keys(document)); + return pyCall(documentProxy.tp$getattr(pyStr.$dir)); }, $flags: { NoArgs: true }, }, diff --git a/src/lib/js.js b/src/lib/js.js deleted file mode 100644 index 416802f2c1..0000000000 --- a/src/lib/js.js +++ /dev/null @@ -1,6 +0,0 @@ -function $builtinmodule() { - const mod = { __name__: new Sk.builtin.str("js") }; - mod.window = Sk.ffi.proxy(window); - Sk.abstr.objectSetItem(Sk.sysmodules, new Sk.builtin.str("js.window"), mod.window); - return mod; -} diff --git a/src/lib/p5.js b/src/lib/p5.js deleted file mode 100644 index a3e8dd712a..0000000000 --- a/src/lib/p5.js +++ /dev/null @@ -1,112 +0,0 @@ -function $builtinmodule() { - const { - builtins: __builtins__, - builtin: { - str: pyStr, - func: pyFunc, - none: { none$: pyNone }, - }, - misceval: { callsimArray: pyCall }, - ffi: { toPy }, - } = Sk; - - // setup a p5 object on Sk if not already there - Sk.p5 || (Sk.p5 = {}); - - const mod = { - __name__: new pyStr("p5"), - p5: toPy(window.p5), - __doc__: new pyStr("A skulpt implementation of the p5 library"), - }; - - // override _start to a plain function and call this in run when we need it - // this is kind of hacky - // _start is set in the constructor and then called - // by overriding the prototype means we can delay the call to _start - // which p5 does on initialization to get the methods in the namespace - let _start; - Object.defineProperty(window.p5.prototype, "_start", { - get() { - return () => {}; - }, - set(val) { - _start = val; - }, - configurable: true, - }); - - function sketch(p) { - // p is a python object since we used a python function - for (let i in window.p5.prototype) { - const asStr = new pyStr(i); - const mangled = asStr.$mangled; - // it would be crazy to override builtins like print - if (!(mangled in __builtins__)) { - mod[mangled] = p.tp$getattr(asStr); - } - } - } - - // create an instance of p5 and assign all the attributes to mod - const p = pyCall(mod.p5, [new pyFunc(sketch), toPy(Sk.p5.node || Sk.canvas)]); - const pInstance = p.valueOf(); - Sk.p5.instance = pInstance; - Sk.p5.kill = pInstance.remove.bind(pInstance); - - const wrapFunc = (func) => () => { - try { - pyCall(func); - } catch (e) { - Sk.uncaughtException && Sk.uncaughtException(e); - } - // note we can't suspend because promises are just ignored in these methods - }; - - mod.run = new pyFunc(function run() { - const main = Sk.sysmodules.quick$lookup(new pyStr("__main__")).$d; - delete window.p5.prototype._start; - pInstance._start = _start; - - [ - "preload", - "setup", - "draw", - "deviceMoved", - "deviceTurned", - "deviceShaken", - "windowResized", - "keyPressed", - "keyReleased", - "keyTyped", - "mousePressed", - "mouseReleased", - "mouseClicked", - "doubleClicked", - "mouseMoved", - "mouseDragged", - "mouseWheel", - "touchStarted", - "touchMoved", - "touchEnded", - ].forEach((methodName) => { - const method = main[methodName]; - if (method !== undefined) { - pInstance[methodName] = wrapFunc(method); - } - }); - - // p5 wants to change the global namespace of things like frameCount, key. So let it - const _setProperty = pInstance._setProperty; - pInstance._setProperty = function (prop, val) { - _setProperty.call(this, prop, val); - const asStr = new pyStr(prop); - const mangled = asStr.$mangled; - mod[mangled] = main[mangled] = p.tp$getattr(asStr); - }; - - pInstance._start(); - return pyNone; - }); - - return mod; -} diff --git a/src/object.js b/src/object.js index 839916d61a..7c68243525 100644 --- a/src/object.js +++ b/src/object.js @@ -159,7 +159,9 @@ Sk.builtin.object = Sk.abstr.buildNativeClass("object", { }, $mergeClassDict(dict) { const classDict = Sk.abstr.lookupAttr(this, Sk.builtin.str.$dict); - dict.dict$merge(classDict); + if (classDict !== undefined) { + dict.dict$merge(classDict); + } const bases = Sk.abstr.lookupAttr(this, Sk.builtin.str.$bases); if (bases === undefined) { return; From 54e4269a15d16610f34a9a0c46b233f6399530a4 Mon Sep 17 00:00:00 2001 From: stu Date: Tue, 22 Nov 2022 14:16:25 +0800 Subject: [PATCH 095/137] implement json with tests --- src/lib/json.js | 625 ++++++++++++++++++++++++++ src/lib/json/__init__.py | 1 - src/lib/json/tests/__init__.py | 1 - test/unit3/test_json.py | 775 +++++++++++++++++++++++++++++++++ 4 files changed, 1400 insertions(+), 2 deletions(-) create mode 100644 src/lib/json.js delete mode 100644 src/lib/json/__init__.py delete mode 100644 src/lib/json/tests/__init__.py create mode 100644 test/unit3/test_json.py diff --git a/src/lib/json.js b/src/lib/json.js new file mode 100644 index 0000000000..ce5fe1d602 --- /dev/null +++ b/src/lib/json.js @@ -0,0 +1,625 @@ +function $builtinmodule() { + const { + builtin: { + str: pyStr, + float_: pyFloat, + list: pyList, + tuple: pyTuple, + dict: pyDict, + func: pyFunc, + TypeError, + ValueError, + NotImplementedError, + sorted, + none: { none$: pyNone }, + bool: { true$: pyTrue, false$: pyFalse }, + checkString, + checkBytes, + }, + ffi: { toPy, toJs, toPyFloat, toPyInt, isTrue }, + abstr: { typeName, buildNativeClass, checkOneArg, setUpModuleMethods, copyKeywordsToNamedArgs }, + misceval: { objectRepr, callsimArray: pyCall }, + } = Sk; + + const json = { + __name__: new pyStr("json"), + __all__: toPy(["dump", "dumps", "load", "loads", "JSONDecoder", "JSONDecodeError", "JSONEncoder"]), + dump: proxyFail("dump"), + load: proxyFail("load"), + JSONDecoder: proxyFail("JSONDecoder"), + JSONEncoder: proxyFail("JSONEncoder"), + }; + + function proxyFail(name) { + return new pyFunc(() => { + throw new NotImplementedError(name + " is not yet implemented in skulpt"); + }); + } + + const attrs = ["msg", "doc", "pos", "lineno", "colno"]; + const JSONDecodeError = (json.JSONDecodeError = buildNativeClass("json.JSONDecodeError", { + base: ValueError, + constructor: function JSONDecodeError(msg, doc, pos) { + const relevant = doc.slice(0, pos); + const lineno = relevant.split("\n").length; + const colno = pos - relevant.lastIndexOf("\n"); + const errmsg = `${msg}: line ${lineno} column ${colno} (char ${pos})`; + ValueError.call(this, errmsg); + this.$msg = msg; + this.$doc = doc; + this.$pos = pos; + this.$lineno = lineno; + this.$colno = colno; + }, + getsets: Object.fromEntries( + attrs.map((attr) => [ + attr, + { + $get() { + return toPy(this["$" + attr]); + }, + }, + ]) + ), + })); + + /********* ENCODING *********/ + + class JSONEncoder { + constructor(skipkeys, ensure_ascii, check_circular, allow_nan, indent, separators, _default, sort_keys) { + this.skipkeys = skipkeys; + this.ensure_ascii = ensure_ascii; + this.check_circular = check_circular; + this.allow_nan = allow_nan; + this.indent = indent; + this.separators = separators; + this.sort_keys = sort_keys; + this.item_separator = ", "; + this.key_separator = ": "; + if (this.separators !== null) { + [this.item_separator, this.key_separator] = this.separators; + } else if (this.indent !== null) { + this.item_separator = ","; + } + if (_default !== null) { + this.default = _default; + } + this.encoder = this.make_encoder(); + } + default(o) { + throw new TypeError(`Object of type ${typeName(o)} is not JSON serializable`); + } + encode(o) { + return new pyStr(this.encoder(o)); + } + make_encoder() { + let markers, _encoder; + if (this.check_circular) { + markers = new Set(); + } else { + markers = null; + } + /** @todo - at the moment we just ignore this */ + if (this.ensure_ascii) { + _encoder = JSON.stringify; + } else { + _encoder = JSON.stringify; + } + const floatstr = (o, allow_nan = this.allow_nan) => { + const v = o.valueOf(); + let text; + if (!Number.isFinite(v)) { + text = v.toString(); + } else { + return objectRepr(o); + } + if (!allow_nan) { + throw new ValueError("Out of range float values are not JSON compliant: " + objectRepr(o)); + } + return text; + }; + return _make_iterencode( + markers, + this.default, + _encoder, + this.indent, + floatstr, + this.key_separator, + this.item_separator, + this.sort_keys, + this.skipkeys + ); + } + } + + const items_str = new pyStr("items"); + + function _make_iterencode( + markers, + _default, + _encoder, + _indent, + _floatstr, + _key_separator, + _item_separator, + _sort_keys, + _skipkeys + ) { + if (_indent !== null && typeof _indent !== "string") { + _indent = " ".repeat(_indent); + } + + let /** @type {() => {}} */ _check_markers, /**@type {() => {}} */ _remove_from_markers; + if (markers !== null) { + _check_markers = (o) => { + if (markers.has(o)) { + throw new ValueError("Circular reference detected"); + } + markers.add(o); + }; + _remove_from_markers = (o) => markers.delete(o); + } else { + _check_markers = (o) => {}; + _remove_from_markers = (o) => {}; + } + + let /** @type {() => {}} */ _initialize_buffer, /** @type {() => {}} */ _finalize_buffer; + if (_indent !== null) { + _initialize_buffer = (buf, _current_indent_level) => { + _current_indent_level += 1; + const newline_indent = "\n" + _indent.repeat(_current_indent_level); + const separator = _item_separator + newline_indent; + buf += newline_indent; + + return [buf, _current_indent_level, separator]; + }; + _finalize_buffer = (buf, ending, _current_indent_level) => { + _current_indent_level -= 1; + buf += "\n" + _indent.repeat(_current_indent_level) + ending; + return buf; + }; + } else { + _initialize_buffer = (buf, _current_indent_level) => [buf, _current_indent_level, _item_separator]; + _finalize_buffer = (buf, ending, _current_indent_level) => buf + ending; + } + + const _unhandled = (o, _current_indent_level) => { + _check_markers(o); + const ret = _iterencode(_default(o), _current_indent_level); + _remove_from_markers(o); + return ret; + }; + + const _iterencode_list = (arr, _current_indent_level) => { + if (!arr.length) { + return "[]"; + } + _check_markers(arr); + let buf, separator; + [buf, _current_indent_level, separator] = _initialize_buffer("[", _current_indent_level); + let first = true; + for (let val of arr) { + if (first) { + first = false; + } else { + buf += separator; + } + buf += _iterencode(val, _current_indent_level); + } + _remove_from_markers(arr); + return _finalize_buffer(buf, "]", _current_indent_level); + }; + + const _iterencode_dict = (dict, _current_indent_level) => { + if (!dict.sq$length()) { + return "{}"; + } + _check_markers(dict); + let buf, separator; + [buf, _current_indent_level, separator] = _initialize_buffer("{", _current_indent_level); + let first = true; + if (_sort_keys) { + const pyItems = pyCall(dict.tp$getattr(items_str)); + const sortedItems = sorted(pyItems); + dict = pyCall(pyDict, [sortedItems]); + } + for (let [key, val] of dict.$items()) { + const k = key.valueOf(); + const type = typeof k; + if (type === "string") { + key = k; + } else if (type === "number") { + key = _floatstr(key); + } else if (type === "boolean" || k === null) { + key = String(k); + } else if (JSBI.__isBigInt(k)) { + key = k.toString(); + } else if (_skipkeys) { + continue; + } else { + throw new TypeError("keys must be str, int, float, bool or None, not " + typeName(key)); + } + if (first) { + first = false; + } else { + buf += separator; + } + buf += _encoder(key); + buf += _key_separator; + buf += _iterencode(val, _current_indent_level); + } + _remove_from_markers(dict); + return _finalize_buffer(buf, "}", _current_indent_level); + }; + + const _iterencode = (o, _current_indent_level = 0) => { + return String( + toJs(o, { + stringHook: (val) => _encoder(val), + numberHook: (val, obj) => _floatstr(obj), + bigintHook: (val) => val.toString(), + dictHook: (dict) => _iterencode_dict(dict, _current_indent_level), + arrayHook: (arr) => _iterencode_list(arr, _current_indent_level), + setHook: (o) => _unhandled(o, _current_indent_level), + funcHook: (val, obj) => _unhandled(obj, _current_indent_level), + objecthook: (val, obj) => _unhandled(obj, _current_indent_level), + unhandledHook: (obj) => _unhandled(obj, _current_indent_level), + }) + ); + }; + + return _iterencode; + } + + const defaultEncoderArgs = [false, true, true, true, null, null, null, false]; + const defaultEncoder = new JSONEncoder(...defaultEncoderArgs); + + /********* DECODER *********/ + + const NUMBER_RE = /(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?/; + + function make_scanner(context) { + const { + parse_object, + parse_array, + parse_string, + parse_float, + parse_int, + parse_constant, + object_hook, + object_pairs_hook, + } = context; + /** + * @param {string} string + * @param {number} idx + */ + const scan_once = (string, idx) => { + const nextchar = string[idx]; + if (nextchar === undefined) { + return [nextchar, idx]; + } + + if (nextchar === '"') { + return parse_string(string, idx + 1); + } else if (nextchar === "{") { + return parse_object(string, idx + 1, scan_once, object_hook, object_pairs_hook); + } else if (nextchar === "[") { + return parse_array(string, idx + 1, scan_once); + } else if (nextchar === "n" && string.substring(idx, idx + 4) === "null") { + return [pyNone, idx + 4]; + } else if (nextchar === "t" && string.substring(idx, idx + 4) === "true") { + return [pyTrue, idx + 4]; + } else if (nextchar === "f" && string.substring(idx, idx + 5) === "false") { + return [pyFalse, idx + 5]; + } + const m = string.substring(idx).match(NUMBER_RE); + + if (m !== null) { + let res; + const [match, integer, frac, exp] = m; + if (frac || exp) { + res = parse_float(integer + (frac || "") + (exp || "")); + } else { + res = parse_int(integer); + } + return [res, idx + match.length]; + } else if (nextchar === "N" && string.substring(idx, idx + 3) === "NaN") { + return [parse_constant("NaN"), idx + 3]; + } else if (nextchar == "I" && string.substring(idx, idx + 8) === "Infinity") { + return [parse_constant("Infinity"), idx + 8]; + } else if (nextchar == "-" && string.substring(idx, idx + 9) === "-Infinity") { + return [parse_constant("-Infinity"), idx + 9]; + } else { + return [undefined, idx]; + } + }; + return scan_once; + } + + // this is taken from tokenize.py - single line string regex + // adjusted to allow multiline strings - but this will get thrown by JSON.parse + const STRING_CHUNK = /"[^"\\]*(?:\\.[^"\\]*)*"/m; + + function scanstring(s, end) { + // get the end of the string literal; + const chunk = s.substring(end - 1).match(STRING_CHUNK); + if (chunk === null) { + throw new JSONDecodeError("Unterminated string starting at", s, end - 1); + } + try { + // just let javascript deal with the string and re throw the error + const string = new pyStr(JSON.parse(chunk[0])); + return [string, end + chunk[0].length - 1]; + } catch (e) { + let pos = e.message.match(/(?:column|position) (\d+)/); + pos = pos && Number(pos[1]); + const offset = e.columnNumber === undefined ? 1 : 2; // firefox reports column, v8 reports postition + end = end + (pos || 0) - offset; + // account for message formatting in v8 and firefox. + const msg = e.message + .replace("JSON.parse: ", "") + .replace(/ at line \d+ column \d+ of the JSON data/, "") + .replace(/ in JSON at position \d+$/, ""); + throw new JSONDecodeError(msg, s, end); + } + } + + const WHITESPACE = /[ \t\n\r]*/; + + function JSONArray(s, end, scan_once) { + const values = []; + let nextchar = s[end]; + const adjust_white_space = () => { + if (nextchar === " " || nextchar === "\t" || nextchar === "\n" || nextchar === "\r") { + const m = s.substring(end).match(WHITESPACE); + end = end + m[0].length; + nextchar = s[end]; + } + }; + adjust_white_space(); + if (nextchar === "]") { + return [new pyList([]), end + 1]; + } + while (true) { + let value; + [value, end] = scan_once(s, end); + if (value === undefined) { + throw new JSONDecodeError("Expecting value", s, end); + } + values.push(value); + nextchar = s[end]; + adjust_white_space(); + end++; + if (nextchar === "]") { + break; + } else if (nextchar !== ",") { + throw new JSONDecodeError("Expecting ',' delimiter", s, end - 1); + } + nextchar = s[end]; + adjust_white_space(); + } + return [new pyList(values), end]; + } + + function JSONObject(s, end, scan_once, object_hook, object_pairs_hook) { + let pairs = []; + let nextchar = s[end]; + const adjust_white_space = () => { + if (nextchar === " " || nextchar === "\t" || nextchar === "\n" || nextchar === "\r") { + const m = s.substring(end).match(WHITESPACE); + end = end + m[0].length; + nextchar = s[end]; + } + }; + if (nextchar !== '"') { + // normally we expect '"' + adjust_white_space(); + if (nextchar === "}") { + if (object_pairs_hook !== null) { + const res = object_pairs_hook(new pyList([])); + return [res, end + 1]; + } + pairs = new pyDict([]); + if (object_hook !== null) { + pairs = object_hook(pairs); + } + return [pairs, end + 1]; + } else if (nextchar !== '"') { + throw new JSONDecodeError("Expecting property name enclosed in double quotes", s, end); + } + } + end += 1; + let key, value; + while (true) { + [key, end] = scanstring(s, end); + /** @todo memo */ + if ((nextchar = s[end]) !== ":") { + adjust_white_space(); + if (s[end] !== ":") { + throw new JSONDecodeError("Expecting ':' delimiter", s, end); + } + } + nextchar = s[++end]; + adjust_white_space(); + [value, end] = scan_once(s, end); + if (value === undefined) { + throw new JSONDecodeError("Expecting value", s, end); + } + nextchar = s[end]; + pairs.push([key, value]); + adjust_white_space(); + end++; + if (nextchar === "}") { + break; + } else if (nextchar !== ",") { + throw new JSONDecodeError("Expecting ',' delimiter", s, end - 1); + } + nextchar = s[end]; + adjust_white_space(); + end++; + if (nextchar !== '"') { + throw new JSONDecodeError("Expecting property name enclosed in double quotes", s, end - 1); + } + } + if (object_pairs_hook !== null) { + const res = object_pairs_hook(new pyList(pairs.map((pair) => new pyTuple(pair)))); + return [res, end]; + } + pairs = new pyDict(pairs.flat()); + if (object_hook !== null) { + pairs = object_hook(pairs); + } + return [pairs, end]; + } + + const _CONSTANTS = { + NaN: new pyFloat(NaN), + Infinity: new pyFloat(Infinity), + "-Infinity": new pyFloat(-Infinity), + }; + + class JSONDecoder { + constructor(object_hook, parse_float, parse_int, parse_constant, object_pairs_hook) { + this.object_hook = object_hook; + this.parse_float = parse_float || toPyFloat; + this.parse_int = parse_int || toPyInt; + this.parse_constant = parse_constant || ((x) => _CONSTANTS[x]); + this.object_pairs_hook = object_pairs_hook; + this.parse_object = JSONObject; + this.parse_array = JSONArray; + this.parse_string = scanstring; + this.scan_once = make_scanner(this); + // we don't use a memo and don't support strict=False + } + white(s, idx) { + const m = (idx === 0 ? s : s.substring(idx)).match(WHITESPACE); + if (m !== null) { + idx += m[0].length; + } + return idx; + } + decode(s) { + s = s.toString(); + let [obj, end] = this.scan_once(s, this.white(s, 0)); + if (obj === undefined) { + throw new JSONDecodeError("Expecting value", s, end); + } + end = this.white(s, end); + if (end !== s.length) { + throw new JSONDecodeError("Extra data", s, end); + } + return obj; + } + } + + const defaultDecoderArgs = Array(5).fill(null); + const defaultDecoder = new JSONDecoder(...defaultDecoderArgs); + + function convertToNullOrFunc(maybeFunc) { + if (maybeFunc === null || maybeFunc === pyNone) { + return null; + } else { + return (o) => pyCall(maybeFunc, [toPy(o)]); + } + } + + setUpModuleMethods("json", json, { + loads: { + $meth(args, kws) { + checkOneArg("dumps", args); + let s = args[0]; + if (checkString(s)) { + // pass + } else if (checkBytes(s)) { + s = new TextDecoder().decode(s.valueOf()); + } else { + throw new TypeError(`the JSON object must be str or bytes, not ${typeName(s)}`); + } + // we ignore cls and don't support **kws + const optionsArray = copyKeywordsToNamedArgs( + "dumps", + ["object_hook", "parse_float", "parse_int", "parse_constant", "object_pairs_hook"], + [], + kws, + defaultDecoderArgs + ).map(convertToNullOrFunc); + + if (optionsArray.every((arg) => arg === null)) { + return defaultDecoder.decode(s); + } + return new JSONDecoder(...optionsArray).decode(s); + }, + $doc: "Deserialize ``s`` (a ``str`` or ``bytes`` instance containing a JSON document) to a Python object.", + $flags: { FastCall: true }, + }, + dumps: { + $meth(args, kws) { + checkOneArg("dumps", args); + const obj = args[0]; + let [skipkeys, ensure_ascii, check_circular, allow_nan, indent, separators, _default, sort_keys] = + copyKeywordsToNamedArgs( + "loads", + [ + "skipkeys", + "ensure_ascii", + "check_circular", + "allow_nan", + "indent", + "separators", + "default", + "sort_keys", + ], + [], + kws, + defaultEncoderArgs + ); + skipkeys = isTrue(skipkeys); + ensure_ascii = isTrue(ensure_ascii); + check_circular = isTrue(check_circular); + allow_nan = isTrue(allow_nan); + indent = toJs(indent); + separators = toJs(separators); + _default = convertToNullOrFunc(_default); + sort_keys = isTrue(sort_keys); + + if ( + !skipkeys && + ensure_ascii && + check_circular && + allow_nan && + indent === null && + separators === null && + _default === null && + !sort_keys + ) { + return defaultEncoder.encode(obj); + } + + if (separators === null) { + } else if ( + !Array.isArray(separators) || + separators.length !== 2 || + typeof separators[0] !== "string" || + typeof separators[1] !== "string" + ) { + throw new TypeError("separators shuld be a list or tuple of strings of length 2"); + } + + return new JSONEncoder( + skipkeys, + ensure_ascii, + check_circular, + allow_nan, + indent, + separators, + _default, + sort_keys + ).encode(obj); + }, + $doc: "Serialize ``obj`` to a JSON formatted ``str``", + $flags: { FastCall: true }, + }, + }); + + return json; +} diff --git a/src/lib/json/__init__.py b/src/lib/json/__init__.py deleted file mode 100644 index 2266aca637..0000000000 --- a/src/lib/json/__init__.py +++ /dev/null @@ -1 +0,0 @@ -import _sk_fail; _sk_fail._("json") diff --git a/src/lib/json/tests/__init__.py b/src/lib/json/tests/__init__.py deleted file mode 100644 index dae43e2387..0000000000 --- a/src/lib/json/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -import _sk_fail; _sk_fail._("tests") diff --git a/test/unit3/test_json.py b/test/unit3/test_json.py new file mode 100644 index 0000000000..f349ea7841 --- /dev/null +++ b/test/unit3/test_json.py @@ -0,0 +1,775 @@ +import unittest +import json + +class PyTest(): + json = json + loads = staticmethod(json.loads) + dumps = staticmethod(json.dumps) + JSONDecodeError = staticmethod(json.JSONDecodeError) + + + +def skulpt_ignore(f): + return lambda self: None + + +class TestDecode(PyTest, unittest.TestCase): + @skulpt_ignore + def test_decimal(self): + rval = self.loads('1.1', parse_float=decimal.Decimal) + self.assertTrue(isinstance(rval, decimal.Decimal)) + self.assertEqual(rval, decimal.Decimal('1.1')) + + def test_float(self): + rval = self.loads('1', parse_int=float) + self.assertTrue(isinstance(rval, float)) + self.assertEqual(rval, 1.0) + + def test_empty_objects(self): + self.assertEqual(self.loads('{}'), {}) + self.assertEqual(self.loads('[]'), []) + self.assertEqual(self.loads('""'), "") + + def test_object_pairs_hook(self): + s = '{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}' + p = [("xkd", 1), ("kcw", 2), ("art", 3), ("hxm", 4), + ("qrt", 5), ("pad", 6), ("hoy", 7)] + self.assertEqual(self.loads(s), eval(s)) + self.assertEqual(self.loads(s, object_pairs_hook=lambda x: x), p) + # self.assertEqual(self.json.load(StringIO(s), + # object_pairs_hook=lambda x: x), p) + od = self.loads(s, object_pairs_hook=OrderedDict) + self.assertEqual(od, OrderedDict(p)) + self.assertEqual(type(od), OrderedDict) + # the object_pairs_hook takes priority over the object_hook + self.assertEqual(self.loads(s, object_pairs_hook=OrderedDict, + object_hook=lambda x: None), + OrderedDict(p)) + # check that empty object literals work (see #17368) + self.assertEqual(self.loads('{}', object_pairs_hook=OrderedDict), + OrderedDict()) + self.assertEqual(self.loads('{"empty": {}}', + object_pairs_hook=OrderedDict), + OrderedDict([('empty', OrderedDict())])) + + def test_decoder_optimizations(self): + # Several optimizations were made that skip over calls to + # the whitespace regex, so this test is designed to try and + # exercise the uncommon cases. The array cases are already covered. + rval = self.loads('{ "key" : "value" , "k":"v" }') + self.assertEqual(rval, {"key":"value", "k":"v"}) + + def check_keys_reuse(self, source, loads): + rval = loads(source) + (a, b), (c, d) = sorted(rval[0]), sorted(rval[1]) + self.assertIs(a, c) + self.assertIs(b, d) + + def test_keys_reuse(self): + s = '[{"a_key": 1, "b_\xe9": 2}, {"a_key": 3, "b_\xe9": 4}]' + self.check_keys_reuse(s, self.loads) + # decoder = self.json.decoder.JSONDecoder() + # self.check_keys_reuse(s, decoder.decode) + # self.assertFalse(decoder.memo) + + def test_extra_data(self): + s = '[1, 2, 3]5' + msg = 'Extra data' + with self.assertRaises(self.JSONDecodeError) as e: + self.loads(s) + self.assertIn(msg, str(e.exception)) + + def test_invalid_escape(self): + s = '["abc\\y"]' + msg = 'escaped' # v8 doesn't give the same error (firefox does) + with self.assertRaises(self.JSONDecodeError) as e: + self.loads(s) + # self.assertIn(msg, str(e.exception)) + + def test_invalid_input_type(self): + msg = 'the JSON object must be str' + for value in [1, 3.14, [], {}, None]: + with self.assertRaises(TypeError) as e: + self.loads(value) + self.assertIn(msg, str(e.exception)) + + @skulpt_ignore + def test_string_with_utf8_bom(self): + # see #18958 + bom_json = "[1,2,3]".encode('utf-8-sig').decode('utf-8') + with self.assertRaises(self.JSONDecodeError) as cm: + self.loads(bom_json) + self.assertIn('BOM', str(cm.exception)) + with self.assertRaises(self.JSONDecodeError) as cm: + self.json.load(StringIO(bom_json)) + self.assertIn('BOM', str(cm.exception)) + # make sure that the BOM is not detected in the middle of a string + bom_in_str = '"{}"'.format(''.encode('utf-8-sig').decode('utf-8')) + self.assertEqual(self.loads(bom_in_str), '\ufeff') + self.assertEqual(self.json.load(StringIO(bom_in_str)), '\ufeff') + + @skulpt_ignore + def test_negative_index(self): + d = self.json.JSONDecoder() + self.assertRaises(ValueError, d.raw_decode, 'a'*42, -50000) + + + +class TestDump(PyTest, unittest.TestCase): + @skulpt_ignore + def test_dump(self): + sio = StringIO() + self.json.dump({}, sio) + self.assertEqual(sio.getvalue(), '{}') + + def test_dumps(self): + self.assertEqual(self.dumps({}), '{}') + + def test_dump_skipkeys(self): + v = {b'invalid_key': False, 'valid_key': True} + with self.assertRaises(TypeError): + self.json.dumps(v) + + s = self.json.dumps(v, skipkeys=True) + o = self.json.loads(s) + self.assertIn('valid_key', o) + self.assertNotIn(b'invalid_key', o) + + def test_encode_truefalse(self): + self.assertEqual(self.dumps( + {True: False, False: True}, sort_keys=True), + '{"false": true, "true": false}') + self.assertEqual(self.dumps( + {2: 3.0, 4.0: 5, False: 1, 6: True}, sort_keys=True), + '{"false": 1, "2": 3.0, "4.0": 5, "6": true}') + + # Issue 16228: Crash on encoding resized list + def test_encode_mutated(self): + a = [object()] * 10 + def crasher(obj): + del a[-1] + self.assertEqual(self.dumps(a, default=crasher), + '[null, null, null, null, null]') + + # Issue 24094 + def test_encode_evil_dict(self): + class D(dict): + def keys(self): + return L + + class X: + def __hash__(self): + del L[0] + return 1337 + + def __lt__(self, o): + return 0 + + L = [X() for i in range(1122)] + d = D() + d[1337] = "true.dat" + self.assertEqual(self.dumps(d, sort_keys=True), '{"1337": "true.dat"}') + +class TestDefault(PyTest, unittest.TestCase): + def test_default(self): + self.assertEqual( + self.dumps(type, default=repr), + self.dumps(repr(type)) + ) + +from collections import OrderedDict + + +CASES = [ + ('/\\"\ucafe\ubabe\uab98\ufcde\ubcda\uef4a\x08\x0c\n\r\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?', '"/\\\\\\"\\ucafe\\ubabe\\uab98\\ufcde\\ubcda\\uef4a\\b\\f\\n\\r\\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?"'), + ('\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), + ('controls', '"controls"'), + ('\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'), + ('{"object with 1 member":["array with 1 element"]}', '"{\\"object with 1 member\\":[\\"array with 1 element\\"]}"'), + (' s p a c e d ', '" s p a c e d "'), + ('\U0001d120', '"\\ud834\\udd20"'), + ('\u03b1\u03a9', '"\\u03b1\\u03a9"'), + ("`1~!@#$%^&*()_+-={':[,]}|;.?", '"`1~!@#$%^&*()_+-={\':[,]}|;.?"'), + ('\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'), + ('\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), +] + +class TestEncodeBasestringAscii(PyTest, unittest.TestCase): + @skulpt_ignore + def test_encode_basestring_ascii(self): + fname = self.json.encoder.encode_basestring_ascii.__name__ + for input_string, expect in CASES: + result = self.json.encoder.encode_basestring_ascii(input_string) + self.assertEqual(result, expect, + '{0!r} != {1!r} for {2}({3!r})'.format( + result, expect, fname, input_string)) + + def test_ordered_dict(self): + # See issue 6105 + items = [('one', 1), ('two', 2), ('three', 3), ('four', 4), ('five', 5)] + s = self.dumps(OrderedDict(items)) + self.assertEqual(s, '{"one": 1, "two": 2, "three": 3, "four": 4, "five": 5}') + + def test_sorted_dict(self): + items = [('one', 1), ('two', 2), ('three', 3), ('four', 4), ('five', 5)] + s = self.dumps(dict(items), sort_keys=True) + self.assertEqual(s, '{"five": 5, "four": 4, "one": 1, "three": 3, "two": 2}') + + +import math + +class TestFloat(PyTest, unittest.TestCase): + def test_floats(self): + for num in [1617161771.7650001, math.pi, math.pi**100, math.pi**-35, 3.1]: + self.assertEqual(float(self.dumps(num)), num) + self.assertEqual(self.loads(self.dumps(num)), num) + + def test_ints(self): + for num in [1, 1<<32, 1<<64]: + self.assertEqual(self.dumps(num), str(num)) + self.assertEqual(int(self.dumps(num)), num) + + def test_out_of_range(self): + self.assertEqual(self.loads('[23456789012E666]'), [float('inf')]) + self.assertEqual(self.loads('[-23456789012E666]'), [float('-inf')]) + + def test_allow_nan(self): + for val in (float('inf'), float('-inf'), float('nan')): + out = self.dumps([val]) + if val == val: # inf + self.assertEqual(self.loads(out), [val]) + else: # nan + res = self.loads(out) + self.assertEqual(len(res), 1) + self.assertNotEqual(res[0], res[0]) + self.assertRaises(ValueError, self.dumps, [val], allow_nan=False) + +import textwrap + +class TestIndent(PyTest, unittest.TestCase): + def test_indent(self): + h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', 'i-vhbjkhnth',{'nifty': 87}, {'field': 'yes', 'morefield': False} ] + + expect = textwrap.dedent("""\ + [ + \t[ + \t\t"blorpie" + \t], + \t[ + \t\t"whoops" + \t], + \t[], + \t"d-shtaeou", + \t"d-nthiouh", + \t"i-vhbjkhnth", + \t{ + \t\t"nifty": 87 + \t}, + \t{ + \t\t"field": "yes", + \t\t"morefield": false + \t} + ]""") + # print(expect) + + d1 = self.dumps(h) + d2 = self.dumps(h, indent=2, sort_keys=True, separators=(',', ': ')) + d3 = self.dumps(h, indent='\t', sort_keys=True, separators=(',', ': ')) + d4 = self.dumps(h, indent=2, sort_keys=True) + d5 = self.dumps(h, indent='\t', sort_keys=True) + + h1 = self.loads(d1) + h2 = self.loads(d2) + h3 = self.loads(d3) + + self.assertEqual(h1, h) + self.assertEqual(h2, h) + self.assertEqual(h3, h) + self.assertEqual(d2, expect.expandtabs(2)) + self.assertEqual(d3, expect) + self.assertEqual(d4, d2) + self.assertEqual(d5, d3) + + def test_indent0(self): + h = {3: 1} + def check(indent, expected): + d1 = self.dumps(h, indent=indent) + self.assertEqual(d1, expected) + + # sio = StringIO() + # self.json.dump(h, sio, indent=indent) + # self.assertEqual(sio.getvalue(), expected) + + # indent=0 should emit newlines + check(0, '{\n"3": 1\n}') + # indent=None is more compact + check(None, '{"3": 1}') + + + + +class TestSeparators(PyTest, unittest.TestCase): + def test_separators(self): + h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', 'i-vhbjkhnth', + {'nifty': 87}, {'field': 'yes', 'morefield': False} ] + + expect = textwrap.dedent("""\ + [ + [ + "blorpie" + ] , + [ + "whoops" + ] , + [] , + "d-shtaeou" , + "d-nthiouh" , + "i-vhbjkhnth" , + { + "nifty" : 87 + } , + { + "field" : "yes" , + "morefield" : false + } + ]""") + + + d1 = self.dumps(h) + d2 = self.dumps(h, indent=2, sort_keys=True, separators=(' ,', ' : ')) + + h1 = self.loads(d1) + h2 = self.loads(d2) + + self.assertEqual(h1, h) + self.assertEqual(h2, h) + self.assertEqual(d2, expect) + + def test_illegal_separators(self): + h = {1: 2, 3: 4} + self.assertRaises(TypeError, self.dumps, h, separators=(b', ', ': ')) + self.assertRaises(TypeError, self.dumps, h, separators=(', ', b': ')) + self.assertRaises(TypeError, self.dumps, h, separators=(b', ', b': ')) + + + + +# from http://json.org/JSON_checker/test/pass1.json +JSON1 = r''' +[ + "JSON Test Pattern pass1", + {"object with 1 member":["array with 1 element"]}, + {}, + [], + -42, + true, + false, + null, + { + "integer": 1234567890, + "real": -9876.543210, + "e": 0.123456789e-12, + "E": 1.234567890E+34, + "": 23456789012E66, + "zero": 0, + "one": 1, + "space": " ", + "quote": "\"", + "backslash": "\\", + "controls": "\b\f\n\r\t", + "slash": "/ & \/", + "alpha": "abcdefghijklmnopqrstuvwyz", + "ALPHA": "ABCDEFGHIJKLMNOPQRSTUVWYZ", + "digit": "0123456789", + "0123456789": "digit", + "special": "`1~!@#$%^&*()_+-={':[,]}|;.?", + "hex": "\u0123\u4567\u89AB\uCDEF\uabcd\uef4A", + "true": true, + "false": false, + "null": null, + "array":[ ], + "object":{ }, + "address": "50 St. James Street", + "url": "http://www.JSON.org/", + "comment": "// /* */": " ", + " s p a c e d " :[1,2 , 3 + +, + +4 , 5 , 6 ,7 ],"compact":[1,2,3,4,5,6,7], + "jsontext": "{\"object with 1 member\":[\"array with 1 element\"]}", + "quotes": "" \u0022 %22 0x22 034 "", + "\/\\\"\uCAFE\uBABE\uAB98\uFCDE\ubcda\uef4A\b\f\n\r\t`1~!@#$%^&*()_+-=[]{}|;:',./<>?" +: "A key can be any string" + }, + 0.5 ,98.6 +, +99.44 +, + +1066, +1e1, +0.1e1, +1e-1, +1e00,2e+00,2e-00 +,"rosebud"] +''' + +class TestPass1(PyTest, unittest.TestCase): + def test_parse(self): + # test in/out equivalence and parsing + res = self.loads(JSON1) + out = self.dumps(res) + self.assertEqual(res, self.loads(out)) + + +# from http://json.org/JSON_checker/test/pass2.json +JSON2 = r''' +[[[[[[[[[[[[[[[[[[["Not too deep"]]]]]]]]]]]]]]]]]]] +''' + +class TestPass2(PyTest, unittest.TestCase): + def test_parse(self): + # test in/out equivalence and parsing + res = self.loads(JSON2) + out = self.dumps(res) + self.assertEqual(res, self.loads(out)) + + +# from http://json.org/JSON_checker/test/pass3.json +JSON3 = r''' +{ + "JSON Test Pattern pass3": { + "The outermost value": "must be an object or array.", + "In this test": "It is an object." + } +} +''' + + +class TestPass3(PyTest, unittest.TestCase): + def test_parse(self): + # test in/out equivalence and parsing + res = self.loads(JSON3) + out = self.dumps(res) + self.assertEqual(res, self.loads(out)) + + +# 2007-10-05 +JSONDOCS = [ + # http://json.org/JSON_checker/test/fail1.json + '"A JSON payload should be an object or array, not a string."', + # http://json.org/JSON_checker/test/fail2.json + '["Unclosed array"', + # http://json.org/JSON_checker/test/fail3.json + '{unquoted_key: "keys must be quoted"}', + # http://json.org/JSON_checker/test/fail4.json + '["extra comma",]', + # http://json.org/JSON_checker/test/fail5.json + '["double extra comma",,]', + # http://json.org/JSON_checker/test/fail6.json + '[ , "<-- missing value"]', + # http://json.org/JSON_checker/test/fail7.json + '["Comma after the close"],', + # http://json.org/JSON_checker/test/fail8.json + '["Extra close"]]', + # http://json.org/JSON_checker/test/fail9.json + '{"Extra comma": true,}', + # http://json.org/JSON_checker/test/fail10.json + '{"Extra value after close": true} "misplaced quoted value"', + # http://json.org/JSON_checker/test/fail11.json + '{"Illegal expression": 1 + 2}', + # http://json.org/JSON_checker/test/fail12.json + '{"Illegal invocation": alert()}', + # http://json.org/JSON_checker/test/fail13.json + '{"Numbers cannot have leading zeroes": 013}', + # http://json.org/JSON_checker/test/fail14.json + '{"Numbers cannot be hex": 0x14}', + # http://json.org/JSON_checker/test/fail15.json + '["Illegal backslash escape: \\x15"]', + # http://json.org/JSON_checker/test/fail16.json + '[\\naked]', + # http://json.org/JSON_checker/test/fail17.json + '["Illegal backslash escape: \\017"]', + # http://json.org/JSON_checker/test/fail18.json + '[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]', + # http://json.org/JSON_checker/test/fail19.json + '{"Missing colon" null}', + # http://json.org/JSON_checker/test/fail20.json + '{"Double colon":: null}', + # http://json.org/JSON_checker/test/fail21.json + '{"Comma instead of colon", null}', + # http://json.org/JSON_checker/test/fail22.json + '["Colon instead of comma": false]', + # http://json.org/JSON_checker/test/fail23.json + '["Bad value", truth]', + # http://json.org/JSON_checker/test/fail24.json + "['single quote']", + # http://json.org/JSON_checker/test/fail25.json + '["\ttab\tcharacter\tin\tstring\t"]', + # http://json.org/JSON_checker/test/fail26.json + '["tab\\ character\\ in\\ string\\ "]', + # http://json.org/JSON_checker/test/fail27.json + '["line\nbreak"]', + # http://json.org/JSON_checker/test/fail28.json + '["line\\\nbreak"]', + # http://json.org/JSON_checker/test/fail29.json + '[0e]', + # http://json.org/JSON_checker/test/fail30.json + '[0e+]', + # http://json.org/JSON_checker/test/fail31.json + '[0e+-1]', + # http://json.org/JSON_checker/test/fail32.json + '{"Comma instead if closing brace": true,', + # http://json.org/JSON_checker/test/fail33.json + '["mismatch"}', + # http://code.google.com/p/simplejson/issues/detail?id=3 + '["A\u001FZ control characters in string"]', +] + +SKIPS = { + 1: "why not have a string payload?", + 18: "spec doesn't specify any nesting limitations", +} + +class TestFail(PyTest, unittest.TestCase): + def test_failures(self): + for idx, doc in enumerate(JSONDOCS): + idx = idx + 1 + if idx in SKIPS: + self.loads(doc) + continue + try: + self.loads(doc) + except self.JSONDecodeError: + pass + else: + self.fail("Expected failure for fail{0}.json: {1!r}".format(idx, doc)) + + def test_non_string_keys_dict(self): + data = {'a' : 1, (1, 2) : 2} + with self.assertRaises(TypeError) as e: + self.dumps(data) + self.assertIn('keys must be str, int, float, bool or None, not tuple', str(e.exception)) + + def test_not_serializable(self): + import sys + with self.assertRaises(TypeError) as e: + self.dumps(sys) + self.assertIn('Object of type module is not JSON serializable', str(e.exception)) + + def test_truncated_input(self): + test_cases = [ + ('', 'Expecting value', 0), + ('[', 'Expecting value', 1), + ('[42', "Expecting ',' delimiter", 3), + ('[42,', 'Expecting value', 4), + ('["', 'Unterminated string starting at', 1), + ('["spam', 'Unterminated string starting at', 1), + ('["spam"', "Expecting ',' delimiter", 7), + ('["spam",', 'Expecting value', 8), + ('{', 'Expecting property name enclosed in double quotes', 1), + ('{"', 'Unterminated string starting at', 1), + ('{"spam', 'Unterminated string starting at', 1), + ('{"spam"', "Expecting ':' delimiter", 7), + ('{"spam":', 'Expecting value', 8), + ('{"spam":42', "Expecting ',' delimiter", 10), + ('{"spam":42,', 'Expecting property name enclosed in double quotes', 11), + ] + test_cases += [ + ('"', 'Unterminated string starting at', 0), + ('"spam', 'Unterminated string starting at', 0), + ] + for data, msg, idx in test_cases: + with self.assertRaises(self.JSONDecodeError) as cm: + self.loads(data) + err = cm.exception + self.assertEqual(err.msg, msg) + self.assertEqual(err.pos, idx) + self.assertEqual(err.lineno, 1) + self.assertEqual(err.colno, idx + 1) + self.assertEqual(str(err), + '%s: line 1 column %d (char %d)' % + (msg, idx + 1, idx)) + + def test_unexpected_data(self): + test_cases = [ + ('[,', 'Expecting value', 1), + ('{"spam":[}', 'Expecting value', 9), + ('[42:', "Expecting ',' delimiter", 3), + ('[42 "spam"', "Expecting ',' delimiter", 4), + ('[42,]', 'Expecting value', 4), + ('{"spam":[42}', "Expecting ',' delimiter", 11), + ('["]', 'Unterminated string starting at', 1), + ('["spam":', "Expecting ',' delimiter", 7), + ('["spam",]', 'Expecting value', 8), + ('{:', 'Expecting property name enclosed in double quotes', 1), + ('{,', 'Expecting property name enclosed in double quotes', 1), + ('{42', 'Expecting property name enclosed in double quotes', 1), + ('[{]', 'Expecting property name enclosed in double quotes', 2), + ('{"spam",', "Expecting ':' delimiter", 7), + ('{"spam"}', "Expecting ':' delimiter", 7), + ('[{"spam"]', "Expecting ':' delimiter", 8), + ('{"spam":}', 'Expecting value', 8), + ('[{"spam":]', 'Expecting value', 9), + ('{"spam":42 "ham"', "Expecting ',' delimiter", 11), + ('[{"spam":42]', "Expecting ',' delimiter", 11), + ('{"spam":42,}', 'Expecting property name enclosed in double quotes', 11), + ] + for data, msg, idx in test_cases: + with self.assertRaises(self.JSONDecodeError) as cm: + self.loads(data) + err = cm.exception + self.assertEqual(err.msg, msg) + self.assertEqual(err.pos, idx) + self.assertEqual(err.lineno, 1) + self.assertEqual(err.colno, idx + 1) + self.assertEqual(str(err), + '%s: line 1 column %d (char %d)' % + (msg, idx + 1, idx)) + + def test_extra_data(self): + test_cases = [ + ('[]]', 'Extra data', 2), + ('{}}', 'Extra data', 2), + ('[],[]', 'Extra data', 2), + ('{},{}', 'Extra data', 2), + ] + test_cases += [ + ('42,"spam"', 'Extra data', 2), + ('"spam",42', 'Extra data', 6), + ] + for data, msg, idx in test_cases: + with self.assertRaises(self.JSONDecodeError) as cm: + self.loads(data) + err = cm.exception + self.assertEqual(err.msg, msg) + self.assertEqual(err.pos, idx) + self.assertEqual(err.lineno, 1) + self.assertEqual(err.colno, idx + 1) + self.assertEqual(str(err), + '%s: line 1 column %d (char %d)' % + (msg, idx + 1, idx)) + + def test_linecol(self): + test_cases = [ + ('!', 1, 1, 0), + (' !', 1, 2, 1), + ('\n!', 2, 1, 1), + ('\n \n\n !', 4, 6, 10), + ] + for data, line, col, idx in test_cases: + with self.assertRaises(self.JSONDecodeError) as cm: + self.loads(data) + err = cm.exception + self.assertEqual(err.msg, 'Expecting value') + self.assertEqual(err.pos, idx) + self.assertEqual(err.lineno, line) + self.assertEqual(err.colno, col) + self.assertEqual(str(err), + 'Expecting value: line %s column %d (char %d)' % + (line, col, idx)) + + +class JSONTestObject: + pass + + +class TestRecursion(PyTest, unittest.TestCase): + def test_listrecursion(self): + x = [] + x.append(x) + try: + self.dumps(x) + except ValueError: + pass + else: + self.fail("didn't raise ValueError on list recursion") + x = [] + y = [x] + x.append(y) + try: + self.dumps(x) + except ValueError: + pass + else: + self.fail("didn't raise ValueError on alternating list recursion") + y = [] + x = [y, y] + # ensure that the marker is cleared + self.dumps(x) + + def test_dictrecursion(self): + x = {} + x["test"] = x + try: + self.dumps(x) + except ValueError: + pass + else: + self.fail("didn't raise ValueError on dict recursion") + x = {} + y = {"a": x, "b": x} + # ensure that the marker is cleared + self.dumps(x) + + @skulpt_ignore + def test_defaultrecursion(self): + class RecursiveJSONEncoder(self.json.JSONEncoder): + recurse = False + def default(self, o): + if o is JSONTestObject: + if self.recurse: + return [JSONTestObject] + else: + return 'JSONTestObject' + return self.json.JSONEncoder.default(o) + + enc = RecursiveJSONEncoder() + self.assertEqual(enc.encode(JSONTestObject), '"JSONTestObject"') + enc.recurse = True + try: + enc.encode(JSONTestObject) + except ValueError: + pass + else: + self.fail("didn't raise ValueError on default recursion") + + + def test_highly_nested_objects_decoding(self): + # test that loading highly-nested objects doesn't segfault when C + # accelerations are used. See #12017 + with self.assertRaises(RecursionError): + self.loads('{"a":' * 100000 + '1' + '}' * 100000) + with self.assertRaises(RecursionError): + self.loads('{"a":' * 100000 + '[1]' + '}' * 100000) + with self.assertRaises(RecursionError): + self.loads('[' * 100000 + '1' + ']' * 100000) + + def test_highly_nested_objects_encoding(self): + # See #12051 + l, d = [], {} + for x in range(100000): + l, d = [l], {'k':d} + with self.assertRaises(RecursionError): + self.dumps(l) + with self.assertRaises(RecursionError): + self.dumps(d) + + @skulpt_ignore + def test_endless_recursion(self): + # See #12051 + class EndlessJSONEncoder(self.json.JSONEncoder): + def default(self, o): + """If check_circular is False, this will keep adding another list.""" + return [o] + + with self.assertRaises(RecursionError): + EndlessJSONEncoder(check_circular=False).encode(5j) + + + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file From 96c7916abecf07f66cea8db2c1c6297d2c2ccd5b Mon Sep 17 00:00:00 2001 From: stu Date: Tue, 22 Nov 2022 14:33:09 +0800 Subject: [PATCH 096/137] fix: error message with an index that's too large --- src/tuple.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tuple.js b/src/tuple.js index 6d1bcd3ef4..3bbf14944d 100644 --- a/src/tuple.js +++ b/src/tuple.js @@ -82,7 +82,7 @@ Sk.builtin.tuple = Sk.abstr.buildNativeClass("tuple", { // sequence and mapping slots mp$subscript(index) { if (Sk.misceval.isIndex(index)) { - let i = Sk.misceval.asIndexSized(index); + let i = Sk.misceval.asIndexSized(index, Sk.builtin.IndexError); if (i < 0) { i = this.v.length + i; } From e263e065cd4d468c3bffb43ef527c0a1e9707d34 Mon Sep 17 00:00:00 2001 From: stu Date: Tue, 22 Nov 2022 23:14:47 +0800 Subject: [PATCH 097/137] pr/fractions --- src/builtin/sys.js | 55 ++- src/complex.js | 13 +- src/float.js | 3 +- src/int.js | 9 + src/lib/fractions.js | 504 +++++++++++++++++++++++ src/lib/math.js | 55 ++- test/unit3/test_fractions.py | 753 +++++++++++++++++++++++++++++++++++ 7 files changed, 1356 insertions(+), 36 deletions(-) create mode 100644 src/lib/fractions.js create mode 100644 test/unit3/test_fractions.py diff --git a/src/builtin/sys.js b/src/builtin/sys.js index 92cc535418..0586da87ad 100644 --- a/src/builtin/sys.js +++ b/src/builtin/sys.js @@ -75,21 +75,21 @@ var $builtinmodule = function (name) { }); const float_info_fields = { - "max": "DBL_MAX -- maximum representable finite float", - "max_exp": "DBL_MAX_EXP -- maximum int e such that radix**(e-1) is representable", - "max_10_exp": "DBL_MAX_10_EXP -- maximum int e such that 10**e is representable", - "min": "DBL_MIN -- Minimum positive normalized float", - "min_exp": "DBL_MIN_EXP -- minimum int e such that radix**(e-1) is a normalized float", - "min_10_exp": "DBL_MIN_10_EXP -- minimum int e such that 10**e is a normalized", - "dig": "DBL_DIG -- digits", - "mant_dig": "DBL_MANT_DIG -- mantissa digits", - "epsilon": "DBL_EPSILON -- Difference between 1 and the next representable float", - "radix": "FLT_RADIX -- radix of exponent", - "rounds": "FLT_ROUNDS -- rounding mode" + max: "DBL_MAX -- maximum representable finite float", + max_exp: "DBL_MAX_EXP -- maximum int e such that radix**(e-1) is representable", + max_10_exp: "DBL_MAX_10_EXP -- maximum int e such that 10**e is representable", + min: "DBL_MIN -- Minimum positive normalized float", + min_exp: "DBL_MIN_EXP -- minimum int e such that radix**(e-1) is a normalized float", + min_10_exp: "DBL_MIN_10_EXP -- minimum int e such that 10**e is a normalized", + dig: "DBL_DIG -- digits", + mant_dig: "DBL_MANT_DIG -- mantissa digits", + epsilon: "DBL_EPSILON -- Difference between 1 and the next representable float", + radix: "FLT_RADIX -- radix of exponent", + rounds: "FLT_ROUNDS -- rounding mode", }; - const float_info_type = Sk.builtin.make_structseq('sys', 'float_info', float_info_fields); - sys.float_info = new float_info_type([ + const FloatInfoType = Sk.builtin.make_structseq("sys", "float_info", float_info_fields); + sys.float_info = new FloatInfoType([ Number.MAX_VALUE, Math.floor(Math.log2(Number.MAX_VALUE)), Math.floor(Math.log10(Number.MAX_VALUE)), @@ -101,14 +101,35 @@ var $builtinmodule = function (name) { Number.EPSILON, 2, 1, - ].map(x => Sk.ffi.remapToPy(x))) + ].map(x => Sk.ffi.remapToPy(x))); const int_info_fields = { bits_per_digit: "size of a digit in bits", sizeof_digit: "size in bytes of the C type used to represent a digit" - } - const int_info_type = Sk.builtin.make_structseq('sys', 'int_info', int_info_fields); - sys.int_info = new int_info_type([30, 4].map((x) => Sk.ffi.remapToPy(x))); + }; + const IntInfoType = Sk.builtin.make_structseq("sys", "int_info", int_info_fields); + sys.int_info = new IntInfoType([30, 4].map((x) => Sk.ffi.remapToPy(x))); + + const hash_info_fields = { + width: "width of the type used for hashing, in bits", + modulus: "prime number giving the modulus on which the hash function is based", + inf: "value to be used for hash of a positive infinity", + nan: "value to be used for hash of a nan", + imag: "multiplier used for the imaginary part of a complex number", + algorithm: "name of the algorithm for hashing of str, bytes and memoryviews", + hash_bits: "internal output size of hash algorithm", + seed_bits: "seed size of hash algorithm", + cutoff: "small string optimization cutoff", + }; + + const HashInfoType = Sk.builtin.make_structseq("sys", "hash_info", hash_info_fields); + + sys.hash_info = new HashInfoType( + [64, JSBI.BigInt("2305843009213693951"), 314159, 0, 1000003, "siphash24", 64, 128, 0].map((x) => + Sk.ffi.remapToPy(x) + ) + ); + sys.__stdout__ = new Sk.builtin.file(new Sk.builtin.str("/dev/stdout"), new Sk.builtin.str("w")); sys.__stdin__ = new Sk.builtin.file(new Sk.builtin.str("/dev/stdin"), new Sk.builtin.str("r")); diff --git a/src/complex.js b/src/complex.js index c3077a3573..247ea2dfd3 100644 --- a/src/complex.js +++ b/src/complex.js @@ -106,7 +106,12 @@ Sk.builtin.complex = Sk.abstr.buildNativeClass("complex", { } return power.call(this, other); }, - + nb$reflected_power(other, z) { + if (z != null && !Sk.builtin.checkNone(z)) { + throw new Sk.builtin.ValueError("complex modulo"); + } + return reflected_power.call(this, other); + }, nb$abs() { const _real = this.real; const _imag = this.imag; @@ -645,14 +650,16 @@ function divide(a_real, a_imag, b_real, b_imag) { return new Sk.builtin.complex(real, imag); } -const power = complexNumberSlot((a_real, a_imag, b_real, b_imag) => { +const _pow = (a_real, a_imag, b_real, b_imag) => { const int_exponent = b_real | 0; // js convert to int if (b_imag === 0.0 && b_real === int_exponent) { return c_powi(a_real, a_imag, int_exponent); } else { return c_pow(a_real, a_imag, b_real, b_imag); } -}); +}; +const power = complexNumberSlot(_pow); +const reflected_power = complexNumberSlot((a_real, a_imag, b_real, b_imag) => _pow(b_real, b_imag, a_real, a_imag)); // power of complex a and complex exponent b function c_pow(a_real, a_imag, b_real, b_imag) { diff --git a/src/float.js b/src/float.js index 13ffe45bca..483e7092c2 100644 --- a/src/float.js +++ b/src/float.js @@ -485,7 +485,8 @@ function remainder(v, w) { function power(v, w) { if (v < 0 && w % 1 !== 0) { - throw new Sk.builtin.ValueError("negative number cannot be raised to a fractional power"); + // let complex deal with it - like CPython does + return new Sk.builtin.complex(v, 0).nb$power(new Sk.builtin.complex(w, 0)); } if (v === 0 && w < 0) { throw new Sk.builtin.ZeroDivisionError("0.0 cannot be raised to a negative power"); diff --git a/src/int.js b/src/int.js index 3bfcc322f4..369dc151e6 100644 --- a/src/int.js +++ b/src/int.js @@ -218,6 +218,14 @@ Sk.builtin.int_ = Sk.abstr.buildNativeClass("int", { }, $doc: "the imaginary part of a complex number", }, + numerator: { + $get: cloneSelf, + }, + denominator: { + $get() { + return INT_ONE; + } + } }, classmethods: { from_bytes: { @@ -986,6 +994,7 @@ for (let i = -5; i < 257; i++) { INTERNED_INT[i] = Object.create(Sk.builtin.int_.prototype, {v: {value: i}}); } const INT_ZERO = INTERNED_INT[0]; +const INT_ONE = INTERNED_INT[1]; // from_bytes and to_bytes function isLittleEndian(byteorder) { diff --git a/src/lib/fractions.js b/src/lib/fractions.js new file mode 100644 index 0000000000..41d9d5b811 --- /dev/null +++ b/src/lib/fractions.js @@ -0,0 +1,504 @@ +// @ts-check +function $builtinmodule(name) { + const mods = {}; + + return Sk.misceval.chain( + Sk.importModule("math", false, true), + (mathMod) => { + mods.math = mathMod; + return Sk.importModule("sys", false, true); + }, + (sysMod) => { + mods.sys = sysMod; + return fractionsMod(mods); + } + ); +} + +function fractionsMod({math, sys}) { + const { + builtin: { + int_: pyInt, + bool: { true$: pyTrue }, + none: { none$: pyNone }, + NotImplemented: { NotImplemented$: pyNotImplemented }, + tuple: pyTuple, + float_: pyFloat, + complex: pyComplex, + str: pyStr, + isinstance: pyIsInstance, + TypeError: pyTypeError, + ZeroDivisionError: pyZeroDivisionError, + ValueError: pyValueError, + NotImplementedError: pyNotImplementedError, + abs: pyAbs, + round: pyRound, + power: pyPower, + }, + ffi: { remapToPy: toPy }, + abstr: { buildNativeClass, copyKeywordsToNamedArgs, numberBinOp, typeName, lookupSpecial, checkArgsLen }, + misceval: { isTrue, richCompareBool, callsimArray: pyCall, objectRepr }, + } = Sk; + + const fractionsMod = { __name__: new pyStr("fractions"), __all__: toPy(["Fraction"]) }; + + const _RATIONAL_FORMAT = + /^\s*(?[-+]?)(?=\d|\.\d)(?\d*)(?:(?:\/(?\d+))?|(?:\.(?\d*))?(?:E(?[-+]?\d+))?)\s*$/i; + const _0 = new pyInt(0); + const _1 = new pyInt(1); + const _2 = new pyInt(2); + const _10 = new pyInt(10); + const _neg_1 = new pyInt(-1); + const _neg_2 = new pyInt(-2); + const s_numerator = new pyStr("numerator"); + const s_denominator = new pyStr("denominator"); + const s_int_ratio = new pyStr("as_integer_ratio"); + const s_from_float = new pyStr("from_float"); + + const getNumer = (x) => x.tp$getattr(s_numerator); + const getDenom = (x) => x.tp$getattr(s_denominator); + + const mul = (a, b) => numberBinOp(a, b, "Mult"); + const div = (a, b) => numberBinOp(a, b, "Div"); + const pow = (a, b) => numberBinOp(a, b, "Pow"); + const add = (a, b) => numberBinOp(a, b, "Add"); + const sub = (a, b) => numberBinOp(a, b, "Sub"); + const floorDiv = (a, b) => numberBinOp(a, b, "FloorDiv"); + const divmod = (a, b) => numberBinOp(a, b, "DivMod"); + const mod = (a, b) => numberBinOp(a, b, "Mod"); + const gcd = math.tp$getattr(new pyStr("gcd")); + + const eq = (a, b) => richCompareBool(a, b, "Eq"); + const lt = (a, b) => richCompareBool(a, b, "Lt"); + const ge = (a, b) => richCompareBool(a, b, "GtE"); + + const METH_NO_ARGS = { NoArgs: true }; + const METH_ONE_ARG = { OneArg: true }; + + const hash_info = sys.tp$getattr(new pyStr("hash_info")); + const _PyHASH_MODULUS = hash_info.tp$getattr(new pyStr("modulus")); + const _PyHASH_INF = hash_info.tp$getattr(new pyStr("inf")); + + function _operator_fallbacks(monomorphic, fallback) { + const forward = function (other) { + if (isTrue(pyIsInstance(other, _NUMBERS_RATIONAL))) { + return monomorphic(this, other); + } else if (other instanceof pyFloat) { + return fallback(this.nb$float(), other); + } else if (other instanceof pyComplex) { + return fallback(pyCall(pyComplex, [this]), other); + } else { + return pyNotImplemented; + } + }; + const reverse = function (other) { + if (isTrue(pyIsInstance(other, _NUMBERS_RATIONAL))) { + return monomorphic(other, this); + } else if (other instanceof pyFloat) { + return fallback(other, this.nb$float()); + } else if (other instanceof pyComplex) { + return fallback(other, pyCall(pyComplex, [this])); + } else { + return pyNotImplemented; + } + }; + return [forward, reverse]; + } + + const [nb$add, nb$reflected_add] = _operator_fallbacks((a, b) => { + const da = getDenom(a); + const db = getDenom(b); + return pyCall(Fraction, [add(mul(getNumer(a), db), mul(getNumer(b), da)), mul(da, db)]); + }, add); + + const [nb$subtract, nb$reflected_subtract] = _operator_fallbacks((a, b) => { + const da = getDenom(a); + const db = getDenom(b); + return pyCall(Fraction, [sub(mul(getNumer(a), db), mul(getNumer(b), da)), mul(da, db)]); + }, sub); + + const [nb$multiply, nb$reflected_multiply] = _operator_fallbacks( + (a, b) => pyCall(Fraction, [mul(getNumer(a), getNumer(b)), mul(getDenom(a), getDenom(b))]), + mul + ); + + const [nb$divide, nb$reflected_divide] = _operator_fallbacks( + (a, b) => pyCall(Fraction, [mul(getNumer(a), getDenom(b)), mul(getDenom(a), getNumer(b))]), + div + ); + + const [nb$floor_divide, nb$reflected_floor_divide] = _operator_fallbacks( + (a, b) => floorDiv(mul(getNumer(a), getDenom(b)), mul(getDenom(a), getNumer(b))), + floorDiv + ); + + const [nb$divmod, nb$reflected_divmod] = _operator_fallbacks((a, b) => { + const da = getDenom(a); + const db = getDenom(b); + const [div, n_mod] = divmod(mul(getNumer(a), db), mul(da, getNumer(b))).valueOf(); + return new pyTuple([div, pyCall(Fraction, [n_mod, mul(da, db)])]); + }, divmod); + + const [nb$remainder, nb$reflected_remainder] = _operator_fallbacks((a, b) => { + const da = getDenom(a); + const db = getDenom(b); + const num = mod(mul(getNumer(a), db), mul(getNumer(b), da)); + return pyCall(Fraction, [num, mul(da, db)]); + }, mod); + + const Fraction = (fractionsMod.Fraction = buildNativeClass("fractions.Fraction", { + constructor: function (numerator, denominator) { + this.$num = numerator || _0; + this.$den = denominator || _1; + }, + slots: { + tp$new(args, kws) { + checkArgsLen("Fraction", args, 0, 2); + + let [numerator, denominator, _normalize] = copyKeywordsToNamedArgs( + "Fraction", + ["numerator", "denominator", "_normalize"], + args, + kws, + [_0, pyNone, pyTrue] + ); + + const self = new this.constructor(); + + if (denominator === pyNone) { + if (numerator.ob$type === pyInt) { + self.$num = numerator; + self.$den = _1; + return self; + } else if (isTrue(pyIsInstance(numerator, _NUMBERS_RATIONAL))) { + self.$num = getNumer(numerator); + self.$den = getDenom(numerator); + return self; + } else if (numerator instanceof pyFloat) { + // todo decimal.Decimal + [self.$num, self.$den] = pyCall(numerator.tp$getattr(s_int_ratio)).valueOf(); + return self; + } else if (numerator instanceof pyStr) { + const s = numerator.toString(); + const m = s.match(_RATIONAL_FORMAT); + if (m === null) { + throw new pyValueError("Invalid literal for Fraction: " + objectRepr(numerator)); + } + numerator = new pyInt(m.groups.num || "0"); + const denom = m.groups.denom; + if (denom) { + denominator = new pyInt(denom); + } else { + denominator = _1; + const decimal = m.groups.decimal; + if (decimal) { + const scale = new pyInt("" + 10 ** decimal.length); + numerator = add(mul(numerator, scale), new pyInt(decimal)); + denominator = mul(denominator, scale); + } + let exp = m.groups.exp; + if (exp) { + exp = new pyInt(exp); + if (lt(exp, _0)) { + denominator = mul(denominator, pow(_10, exp.nb$negative())); + } else { + numerator = mul(numerator, pow(_10, exp)); + } + } + } + if (m.groups.sign == "-") { + numerator = numerator.nb$negative(); + } + } else { + throw new pyTypeError("argument should be a string or a Rational instance"); + } + } else if (numerator.ob$type === pyInt && denominator.ob$type === pyInt) { + // normal case pass + } else if ( + isTrue(pyIsInstance(numerator, _NUMBERS_RATIONAL)) && + isTrue(pyIsInstance(denominator, _NUMBERS_RATIONAL)) + ) { + [numerator, denominator] = [ + mul(getNumer(numerator), getDenom(denominator)), + mul(getNumer(denominator), getDenom(numerator)), + ]; + } else { + throw new pyTypeError("both arguments should be Rational instances"); + } + + if (eq(denominator, _0)) { + throw new pyZeroDivisionError(`Fraction(${numerator}, 0)`); + } + if (isTrue(_normalize)) { + let g = pyCall(gcd, [numerator, denominator]); + if (lt(denominator, _0)) { + g = g.nb$negative(); + } + numerator = floorDiv(numerator, g); + denominator = floorDiv(denominator, g); + } + + self.$num = numerator; + self.$den = denominator; + return self; + }, + $r() { + const name = lookupSpecial(this.ob$type, pyStr.$name); + return new pyStr(`${name}(${this.$num}, ${this.$den})`); + }, + tp$str() { + if (eq(this.$den, _1)) { + return new pyStr(this.$num); + } + return new pyStr(`${this.$num}/${this.$den}`); + }, + tp$hash() { + const dinv = pyPower(this.$den, sub(_PyHASH_MODULUS, 2), _PyHASH_MODULUS); + let hash_; + if (!isTrue(dinv)) { + hash_ = _PyHASH_INF; + } else { + hash_ = mod(mul(pyAbs(this.$num), dinv), _PyHASH_MODULUS); + } + const rv = ge(this, _0) ? hash_ : hash_.nb$negative(); + return eq(rv, _neg_1) ? _neg_2 : rv; + }, + tp$richcompare(other, OP) { + const op = (a, b) => richCompareBool(a, b, OP); + + if (OP === "Eq" || OP == "NotEq") { + if (other.ob$type === pyInt) { + const rv = eq(this.$num, other) && eq(this.$den, _1); + return OP === "Eq" ? rv : !rv; + } + if (other instanceof Fraction || other instanceof pyInt) { + const rv = eq(this.$num, getNumer(other)) && eq(this.$den, getDenom(other)); + return OP === "Eq" ? rv : !rv; + } + if (other instanceof pyComplex) { + if (eq(other.tp$getattr(new pyStr("imag")), _0)) { + other = other.tp$getattr(new pyStr("real")); + } + } + } + + if (isTrue(pyIsInstance(other, _NUMBERS_RATIONAL))) { + return op(mul(getNumer(this), getDenom(other)), mul(getDenom(this), getNumer(other))); + } + if (other instanceof pyFloat) { + if (!Number.isFinite(other.valueOf())) { + return op(new pyFloat(0), other); + } else { + return op(this, pyCall(this.tp$getattr(s_from_float), [other])); + } + } + return pyNotImplemented; + }, + + tp$as_number: true, + nb$add, + nb$reflected_add, + nb$subtract, + nb$reflected_subtract, + nb$multiply, + nb$reflected_multiply, + nb$divide, + nb$reflected_divide, + nb$floor_divide, + nb$reflected_floor_divide, + nb$divmod, + nb$reflected_divmod, + nb$remainder, + nb$reflected_remainder, + nb$power(other) { + if (isTrue(pyIsInstance(other, _NUMBERS_RATIONAL))) { + if (eq(getDenom(other), _1)) { + let power = getNumer(other); + if (ge(power, _0)) { + return pyCall(Fraction, [pow(this.$num, power), pow(this.$den, power)]); + } else if (ge(this.$num, _0)) { + power = power.nb$negative(); + return pyCall(Fraction, [pow(this.$den, power), pow(this.$num, power)]); + } else { + power = power.nb$negative(); + return pyCall(Fraction, [ + pow(this.$den.nb$negative(), power), + pow(this.$num.nb$negative(), power), + ]); + } + } else { + return pow(this.nb$float(), pyCall(pyFloat, [other])); + } + } else { + return pow(this.nb$float(), other); + } + }, + nb$reflected_power(other) { + if (eq(this.$den, _1) && ge(this.$num, _0)) { + return pow(other, this.$num); + } + + if (isTrue(pyIsInstance(other, _NUMBERS_RATIONAL))) { + return pow(new Fraction(getNumer(other), getDenom(other)), this); + } + + if (eq(this.$den, _1)) { + return pow(other, this.$num); + } + + return pow(other, this.nb$float()); + }, + nb$positive() { + return new Fraction(this.$num, this.$den); + }, + nb$negative() { + return new Fraction(this.$num.nb$negative(), this.$den); + }, + nb$abs() { + return new Fraction(this.$num.nb$abs(), this.$den); + }, + nb$bool() { + return this.$num.nb$bool(); + }, + nb$float() { + return div(this.$num, this.$den); + }, + }, + methods: { + as_integer_ratio: { + $meth() { + return new pyTuple([this.$num, this.$den]); + }, + $flags: METH_NO_ARGS, + }, + limit_denominator: {}, + __trunc__: { + $meth() { + if (lt(this.$num, _0)) { + return floorDiv(this.$num.nb$negative(), this.$den).nb$negative(); + } else { + return floorDiv(this.$num, this.$den); + } + }, + $flags: METH_NO_ARGS, + }, + __floor__: { + $meth() { + return floorDiv(this.$num, this.$den); + }, + $flags: METH_NO_ARGS, + }, + __ceil__: { + $meth() { + return floorDiv(this.$num.nb$negative(), this.$den).nb$negative(); + }, + $flags: METH_NO_ARGS, + }, + __round__: { + $meth(ndigits) { + if (ndigits === pyNone) { + const [floor, rem] = divmod(this.$num, this.$den).valueOf(); + const doub = mul(rem, _2); + if (lt(doub, this.$den)) { + return floor; + } else if (lt(this.$den, doub)) { + return add(floor, _1); + } else if (eq(mod(floor, _2), _0)) { + return floor; + } else { + return add(floor, _1); + } + } + + const shift = pow(_10, pyAbs(ndigits)); + + if (lt(_0, ndigits)) { + return pyCall(Fraction, [pyRound(mul(this, shift)), shift]); + } else { + return pyCall(Fraction, [mul(pyRound(div(this, shift)), shift)]); + } + }, + $flags: { NamedArgs: ["ndigits"], Defaults: [pyNone] }, + }, + __reduce__: { + $meth() { + return new pyTuple([this.ob$type, new pyTuple([new pyStr(this)])]); + }, + $flags: METH_NO_ARGS, + }, + __copy__: { + $meth() { + if (this.ob$type === Fraction) { + return this; + } + return pyCall(this.ob$type, [this.$num, this.$den]); + }, + $flags: METH_NO_ARGS, + }, + __deepcopy__: { + $meth(_memo) { + if (this.ob$type === Fraction) { + return this; + } + return pyCall(this.ob$type, [this.$num, this.$den]); + }, + $flags: METH_ONE_ARG, + }, + }, + classmethods: { + from_float: { + $meth(f) { + if (f instanceof pyInt) { + return pyCall(this, [f]); + } else if (!(f instanceof pyFloat)) { + throw new pyTypeError( + `${typeName(this)}.from_float() only takes floats, not ${objectRepr(f)}, (${typeName(f)})` + ); + } else { + const [num, den] = pyCall(f.tp$getattr(s_int_ratio)).valueOf(); + return pyCall(this, [num, den]); + } + }, + $flags: METH_ONE_ARG, + }, + from_decimal: { + $meth() { + throw pyNotImplementedError("from_decimal not yet implemented in SKulpt"); + }, + $flags: METH_ONE_ARG, + }, + }, + getsets: { + numerator: { + $get() { + return this.$num; + }, + }, + denominator: { + $get() { + return this.$den; + }, + }, + _numerator: { + $get() { + return this.$num; + }, + $set(v) { + this.$num = v; + }, + }, + _denominator: { + $get() { + return this.$den; + }, + $set(v) { + this.$den = v; + }, + }, + }, + })); + + const _NUMBERS_RATIONAL = new pyTuple([pyInt, Fraction]); + + return fractionsMod; +} diff --git a/src/lib/math.js b/src/lib/math.js index bcb523efce..1abbf0f5b6 100644 --- a/src/lib/math.js +++ b/src/lib/math.js @@ -1,4 +1,11 @@ const $builtinmodule = function (name) { + + const { + builtin: { str: pyStr, float_: pyFloat, TypeError: pyTypeError, pyCheckType, checkNumber }, + abstr: { lookupSpecial }, + misceval: { callsimOrSuspendArray: pyCallOrSuspend }, + } = Sk; + const math = { // Mathematical Constants pi: new Sk.builtin.float_(Math.PI), @@ -9,13 +16,21 @@ const $builtinmodule = function (name) { }; // Number-theoretic and representation functions + const s_ceil = new pyStr("__ceil__"); + function ceil(x) { - Sk.builtin.pyCheckType("", "real number", Sk.builtin.checkNumber(x)); - const _x = Sk.builtin.asnum$(x); - if (Sk.__future__.ceil_floor_int) { - return new Sk.builtin.int_(Math.ceil(_x)); + let _x; + if (x.ob$type !== pyFloat) { + const method = lookupSpecial(x, s_ceil); + if (method !== undefined) { + return pyCallOrSuspend(method); + } + pyCheckType("", "real number", checkNumber(x)); + _x = Sk.builtin.asnum$(x); + } else { + _x = x.v; } - return new Sk.builtin.float_(Math.ceil(_x)); + return new pyFloat(Math.ceil(_x)); }; function comb(n, k) { @@ -120,14 +135,21 @@ const $builtinmodule = function (name) { } }; - function floor(x) { - Sk.builtin.pyCheckType("x", "number", Sk.builtin.checkNumber(x)); + const s_floor = new pyStr("__floor__"); - if (Sk.__future__.ceil_floor_int) { - return new Sk.builtin.int_(Math.floor(Sk.builtin.asnum$(x))); + function floor(x) { + let _x; + if (x.ob$type === pyFloat) { + _x = x.v; + } else { + const method = lookupSpecial(x, s_floor); + if (method !== undefined) { + return pyCallOrSuspend(method); + } + pyCheckType("x", "number", checkNumber(x)); + _x = Sk.builtin.asnum$(x); } - - return new Sk.builtin.float_(Math.floor(Sk.builtin.asnum$(x))); + return new pyFloat(Math.floor(_x)); }; function fmod(x, y) { @@ -652,11 +674,14 @@ const $builtinmodule = function (name) { }; function trunc(x) { - Sk.builtin.pyCheckType("x", "number", Sk.builtin.checkNumber(x)); - if (Sk.builtin.checkInt(x)) { - return x; //deals with large ints being passed + if (x.ob$type === pyFloat) { + return x.nb$int(); + } + const truncFn = lookupSpecial(x, pyStr.$trunc); + if (truncFn === undefined) { + throw new pyTypeError(`type ${x.tp$name} doesn't define __trunc__ method`); } - return new Sk.builtin.int_(Sk.builtin.asnum$(x) | 0); + return pyCallOrSuspend(truncFn); }; // Power and logarithmic functions diff --git a/test/unit3/test_fractions.py b/test/unit3/test_fractions.py new file mode 100644 index 0000000000..c52d3f5600 --- /dev/null +++ b/test/unit3/test_fractions.py @@ -0,0 +1,753 @@ +"""Tests for Lib/fractions.py.""" + +# from decimal import Decimal +# from test.support import requires_IEEE_754 +import math +# import numbers +import operator +import fractions +import functools +import sys +import unittest +from copy import copy, deepcopy +# from pickle import dumps, loads +F = fractions.Fraction + +def skip_skulpt(f): + @functools.wraps(f) + def skip(self): + return + return skip + + +# HACK for missing numbers module +ModType = type(sys) + +numbers = ModType("numbers") + +class Rational_(tuple): + def register(self, cls): + numbers.Rational = self + (cls,) +numbers.Rational = Rational_([int, F]) + + +class DummyFloat(object): + """Dummy float class for testing comparisons with Fractions""" + + def __init__(self, value): + if not isinstance(value, float): + raise TypeError("DummyFloat can only be initialized from float") + self.value = value + + def _richcmp(self, other, op): + if isinstance(other, numbers.Rational): + return op(F.from_float(self.value), other) + elif isinstance(other, DummyFloat): + return op(self.value, other.value) + else: + return NotImplemented + + def __eq__(self, other): return self._richcmp(other, operator.eq) + def __le__(self, other): return self._richcmp(other, operator.le) + def __lt__(self, other): return self._richcmp(other, operator.lt) + def __ge__(self, other): return self._richcmp(other, operator.ge) + def __gt__(self, other): return self._richcmp(other, operator.gt) + + # shouldn't be calling __float__ at all when doing comparisons + def __float__(self): + assert False, "__float__ should not be invoked for comparisons" + + # same goes for subtraction + def __sub__(self, other): + assert False, "__sub__ should not be invoked for comparisons" + __rsub__ = __sub__ + + +class DummyRational(object): + """Test comparison of Fraction with a naive rational implementation.""" + + def __init__(self, num, den): + g = math.gcd(num, den) + self.num = num // g + self.den = den // g + + def __eq__(self, other): + if isinstance(other, fractions.Fraction): + return (self.num == other._numerator and + self.den == other._denominator) + else: + return NotImplemented + + def __lt__(self, other): + return(self.num * other._denominator < self.den * other._numerator) + + def __gt__(self, other): + return(self.num * other._denominator > self.den * other._numerator) + + def __le__(self, other): + return(self.num * other._denominator <= self.den * other._numerator) + + def __ge__(self, other): + return(self.num * other._denominator >= self.den * other._numerator) + + # this class is for testing comparisons; conversion to float + # should never be used for a comparison, since it loses accuracy + def __float__(self): + assert False, "__float__ should not be invoked" + +class DummyFraction(fractions.Fraction): + """Dummy Fraction subclass for copy and deepcopy testing.""" + + +def _components(r): + return (r.numerator, r.denominator) + + +class FractionTest(unittest.TestCase): + + assertTupleEqual = unittest.TestCase.assertEqual + assertListEqual = unittest.TestCase.assertEqual + + def assertTypedEquals(self, expected, actual): + """Asserts that both the types and values are the same.""" + self.assertEqual(type(expected), type(actual)) + self.assertEqual(expected, actual) + + def assertTypedTupleEquals(self, expected, actual): + """Asserts that both the types and values in the tuples are the same.""" + self.assertTupleEqual(expected, actual) + self.assertListEqual(list(map(type, expected)), list(map(type, actual))) + + def assertRaisesMessage(self, exc_type, message, + callable, *args, **kwargs): + """Asserts that callable(*args, **kwargs) raises exc_type(message).""" + try: + callable(*args, **kwargs) + except exc_type as e: + self.assertEqual(message, str(e)) + else: + self.fail("%s not raised" % exc_type.__name__) + + def testInit(self): + self.assertEqual((0, 1), _components(F())) + self.assertEqual((7, 1), _components(F(7))) + self.assertEqual((7, 3), _components(F(F(7, 3)))) + + self.assertEqual((-1, 1), _components(F(-1, 1))) + self.assertEqual((-1, 1), _components(F(1, -1))) + self.assertEqual((1, 1), _components(F(-2, -2))) + self.assertEqual((1, 2), _components(F(5, 10))) + self.assertEqual((7, 15), _components(F(7, 15))) + self.assertEqual((10**23, 1), _components(F(10**23))) + + self.assertEqual((3, 77), _components(F(F(3, 7), 11))) + self.assertEqual((-9, 5), _components(F(2, F(-10, 9)))) + self.assertEqual((2486, 2485), _components(F(F(22, 7), F(355, 113)))) + + self.assertRaisesMessage(ZeroDivisionError, "Fraction(12, 0)", + F, 12, 0) + self.assertRaises(TypeError, F, 1.5 + 3j) + + self.assertRaises(TypeError, F, "3/2", 3) + self.assertRaises(TypeError, F, 3, 0j) + self.assertRaises(TypeError, F, 3, 1j) + self.assertRaises(TypeError, F, 1, 2, 3) + + # @requires_IEEE_754 + def testInitFromFloat(self): + self.assertEqual((5, 2), _components(F(2.5))) + self.assertEqual((0, 1), _components(F(-0.0))) + self.assertEqual((3602879701896397, 36028797018963968), + _components(F(0.1))) + # bug 16469: error types should be consistent with float -> int + self.assertRaises(ValueError, F, float('nan')) + self.assertRaises(OverflowError, F, float('inf')) + self.assertRaises(OverflowError, F, float('-inf')) + + @skip_skulpt + def testInitFromDecimal(self): + self.assertEqual((11, 10), + _components(F(Decimal('1.1')))) + self.assertEqual((7, 200), + _components(F(Decimal('3.5e-2')))) + self.assertEqual((0, 1), + _components(F(Decimal('.000e20')))) + # bug 16469: error types should be consistent with decimal -> int + self.assertRaises(ValueError, F, Decimal('nan')) + self.assertRaises(ValueError, F, Decimal('snan')) + self.assertRaises(OverflowError, F, Decimal('inf')) + self.assertRaises(OverflowError, F, Decimal('-inf')) + + def testFromString(self): + self.assertEqual((5, 1), _components(F("5"))) + self.assertEqual((3, 2), _components(F("3/2"))) + self.assertEqual((3, 2), _components(F(" \n +3/2"))) + self.assertEqual((-3, 2), _components(F("-3/2 "))) + self.assertEqual((13, 2), _components(F(" 013/02 \n "))) + self.assertEqual((16, 5), _components(F(" 3.2 "))) + self.assertEqual((-16, 5), _components(F(" -3.2 "))) + self.assertEqual((-3, 1), _components(F(" -3. "))) + self.assertEqual((3, 5), _components(F(" .6 "))) + self.assertEqual((1, 3125), _components(F("32.e-5"))) + self.assertEqual((1000000, 1), _components(F("1E+06"))) + self.assertEqual((-12300, 1), _components(F("-1.23e4"))) + self.assertEqual((0, 1), _components(F(" .0e+0\t"))) + self.assertEqual((0, 1), _components(F("-0.000e0"))) + + self.assertRaisesMessage( + ZeroDivisionError, "Fraction(3, 0)", + F, "3/0") + self.assertRaisesMessage( + ValueError, "Invalid literal for Fraction: '3/'", + F, "3/") + self.assertRaisesMessage( + ValueError, "Invalid literal for Fraction: '/2'", + F, "/2") + self.assertRaisesMessage( + ValueError, "Invalid literal for Fraction: '3 /2'", + F, "3 /2") + self.assertRaisesMessage( + # Denominators don't need a sign. + ValueError, "Invalid literal for Fraction: '3/+2'", + F, "3/+2") + self.assertRaisesMessage( + # Imitate float's parsing. + ValueError, "Invalid literal for Fraction: '+ 3/2'", + F, "+ 3/2") + self.assertRaisesMessage( + # Avoid treating '.' as a regex special character. + ValueError, "Invalid literal for Fraction: '3a2'", + F, "3a2") + self.assertRaisesMessage( + # Don't accept combinations of decimals and rationals. + ValueError, "Invalid literal for Fraction: '3/7.2'", + F, "3/7.2") + self.assertRaisesMessage( + # Don't accept combinations of decimals and rationals. + ValueError, "Invalid literal for Fraction: '3.2/7'", + F, "3.2/7") + self.assertRaisesMessage( + # Allow 3. and .3, but not . + ValueError, "Invalid literal for Fraction: '.'", + F, ".") + + def testImmutable(self): + r = F(7, 3) + r.__init__(2, 15) + self.assertEqual((7, 3), _components(r)) + + self.assertRaises(AttributeError, setattr, r, 'numerator', 12) + self.assertRaises(AttributeError, setattr, r, 'denominator', 6) + self.assertEqual((7, 3), _components(r)) + + # But if you _really_ need to: + r._numerator = 4 + r._denominator = 2 + self.assertEqual((4, 2), _components(r)) + # Which breaks some important operations: + self.assertNotEqual(F(4, 2), r) + + def testFromFloat(self): + self.assertRaises(TypeError, F.from_float, 3+4j) + self.assertEqual((10, 1), _components(F.from_float(10))) + bigint = 1234567890123456789 + self.assertEqual((bigint, 1), _components(F.from_float(bigint))) + self.assertEqual((0, 1), _components(F.from_float(-0.0))) + self.assertEqual((10, 1), _components(F.from_float(10.0))) + self.assertEqual((-5, 2), _components(F.from_float(-2.5))) + self.assertEqual((99999999999999991611392, 1), + _components(F.from_float(1e23))) + self.assertEqual(float(10**23), float(F.from_float(1e23))) + self.assertEqual((3602879701896397, 1125899906842624), + _components(F.from_float(3.2))) + self.assertEqual(3.2, float(F.from_float(3.2))) + + inf = 1e1000 + nan = inf - inf + # bug 16469: error types should be consistent with float -> int + self.assertRaisesMessage( + OverflowError, "cannot convert Infinity to integer ratio", + F.from_float, inf) + self.assertRaisesMessage( + OverflowError, "cannot convert Infinity to integer ratio", + F.from_float, -inf) + self.assertRaisesMessage( + ValueError, "cannot convert NaN to integer ratio", + F.from_float, nan) + + @skip_skulpt + def testFromDecimal(self): + self.assertRaises(TypeError, F.from_decimal, 3+4j) + self.assertEqual(F(10, 1), F.from_decimal(10)) + self.assertEqual(F(0), F.from_decimal(Decimal("-0"))) + self.assertEqual(F(5, 10), F.from_decimal(Decimal("0.5"))) + self.assertEqual(F(5, 1000), F.from_decimal(Decimal("5e-3"))) + self.assertEqual(F(5000), F.from_decimal(Decimal("5e3"))) + self.assertEqual(1 - F(1, 10**30), + F.from_decimal(Decimal("0." + "9" * 30))) + + # bug 16469: error types should be consistent with decimal -> int + self.assertRaisesMessage( + OverflowError, "cannot convert Infinity to integer ratio", + F.from_decimal, Decimal("inf")) + self.assertRaisesMessage( + OverflowError, "cannot convert Infinity to integer ratio", + F.from_decimal, Decimal("-inf")) + self.assertRaisesMessage( + ValueError, "cannot convert NaN to integer ratio", + F.from_decimal, Decimal("nan")) + self.assertRaisesMessage( + ValueError, "cannot convert NaN to integer ratio", + F.from_decimal, Decimal("snan")) + + def test_as_integer_ratio(self): + self.assertEqual(F(4, 6).as_integer_ratio(), (2, 3)) + self.assertEqual(F(-4, 6).as_integer_ratio(), (-2, 3)) + self.assertEqual(F(4, -6).as_integer_ratio(), (-2, 3)) + self.assertEqual(F(0, 6).as_integer_ratio(), (0, 1)) + + def testLimitDenominator(self): + rpi = F('3.1415926535897932') + self.assertEqual(rpi.limit_denominator(10000), F(355, 113)) + self.assertEqual(-rpi.limit_denominator(10000), F(-355, 113)) + self.assertEqual(rpi.limit_denominator(113), F(355, 113)) + self.assertEqual(rpi.limit_denominator(112), F(333, 106)) + self.assertEqual(F(201, 200).limit_denominator(100), F(1)) + self.assertEqual(F(201, 200).limit_denominator(101), F(102, 101)) + self.assertEqual(F(0).limit_denominator(10000), F(0)) + for i in (0, -1): + self.assertRaisesMessage( + ValueError, "max_denominator should be at least 1", + F(1).limit_denominator, i) + + def testConversions(self): + self.assertTypedEquals(-1, math.trunc(F(-11, 10))) + self.assertTypedEquals(1, math.trunc(F(11, 10))) + self.assertTypedEquals(-2, math.floor(F(-11, 10))) + self.assertTypedEquals(-1, math.ceil(F(-11, 10))) + self.assertTypedEquals(-1, math.ceil(F(-10, 10))) + self.assertTypedEquals(-1, int(F(-11, 10))) + self.assertTypedEquals(0, round(F(-1, 10))) + self.assertTypedEquals(0, round(F(-5, 10))) + self.assertTypedEquals(-2, round(F(-15, 10))) + self.assertTypedEquals(-1, round(F(-7, 10))) + + self.assertEqual(False, bool(F(0, 1))) + self.assertEqual(True, bool(F(3, 2))) + self.assertTypedEquals(0.1, float(F(1, 10))) + + # Check that __float__ isn't implemented by converting the + # numerator and denominator to float before dividing. + self.assertRaises(OverflowError, float, int('2'*400+'7')) + self.assertAlmostEqual(2.0/3, + float(F(int('2'*400+'7'), int('3'*400+'1')))) + + self.assertTypedEquals(0.1+0j, complex(F(1,10))) + + @skip_skulpt + def testBoolGuarateesBoolReturn(self): + # Ensure that __bool__ is used on numerator which guarantees a bool + # return. See also bpo-39274. + @functools.total_ordering + class CustomValue: + denominator = 1 + + def __init__(self, value): + self.value = value + + def __bool__(self): + return bool(self.value) + + @property + def numerator(self): + # required to preserve `self` during instantiation + return self + + def __eq__(self, other): + raise AssertionError("Avoid comparisons in Fraction.__bool__") + + __lt__ = __eq__ + + # We did not implement all abstract methods, so register: + numbers.Rational.register(CustomValue) + + numerator = CustomValue(1) + r = F(numerator) + # ensure the numerator was not lost during instantiation: + self.assertIs(r.numerator, numerator) + self.assertIs(bool(r), True) + + numerator = CustomValue(0) + r = F(numerator) + self.assertIs(bool(r), False) + + def testRound(self): + self.assertTypedEquals(F(-200), round(F(-150), -2)) + self.assertTypedEquals(F(-200), round(F(-250), -2)) + self.assertTypedEquals(F(30), round(F(26), -1)) + self.assertTypedEquals(F(-2, 10), round(F(-15, 100), 1)) + self.assertTypedEquals(F(-2, 10), round(F(-25, 100), 1)) + + def testArithmetic(self): + self.assertEqual(F(1, 2), F(1, 10) + F(2, 5)) + self.assertEqual(F(-3, 10), F(1, 10) - F(2, 5)) + self.assertEqual(F(1, 25), F(1, 10) * F(2, 5)) + self.assertEqual(F(1, 4), F(1, 10) / F(2, 5)) + self.assertTypedEquals(2, F(9, 10) // F(2, 5)) + self.assertTypedEquals(10**23, F(10**23, 1) // F(1)) + self.assertEqual(F(5, 6), F(7, 3) % F(3, 2)) + self.assertEqual(F(2, 3), F(-7, 3) % F(3, 2)) + self.assertEqual((F(1), F(5, 6)), divmod(F(7, 3), F(3, 2))) + self.assertEqual((F(-2), F(2, 3)), divmod(F(-7, 3), F(3, 2))) + self.assertEqual(F(8, 27), F(2, 3) ** F(3)) + self.assertEqual(F(27, 8), F(2, 3) ** F(-3)) + self.assertTypedEquals(2.0, F(4) ** F(1, 2)) + self.assertEqual(F(1, 1), +F(1, 1)) + z = pow(F(-1), F(1, 2)) + self.assertAlmostEqual(z.real, 0) + self.assertEqual(z.imag, 1) + # Regression test for #27539. + p = F(-1, 2) ** 0 + self.assertEqual(p, F(1, 1)) + self.assertEqual(p.numerator, 1) + self.assertEqual(p.denominator, 1) + p = F(-1, 2) ** -1 + self.assertEqual(p, F(-2, 1)) + self.assertEqual(p.numerator, -2) + self.assertEqual(p.denominator, 1) + p = F(-1, 2) ** -2 + self.assertEqual(p, F(4, 1)) + self.assertEqual(p.numerator, 4) + self.assertEqual(p.denominator, 1) + + def testLargeArithmetic(self): + self.assertTypedEquals( + F(10101010100808080808080808101010101010000000000000000, + 1010101010101010101010101011111111101010101010101010101010101), + F(10**35+1, 10**27+1) % F(10**27+1, 10**35-1) + ) + self.assertTypedEquals( + F(7, 1901475900342344102245054808064), + F(-2**100, 3) % F(5, 2**100) + ) + self.assertTypedTupleEquals( + (9999999999999999, + F(10101010100808080808080808101010101010000000000000000, + 1010101010101010101010101011111111101010101010101010101010101)), + divmod(F(10**35+1, 10**27+1), F(10**27+1, 10**35-1)) + ) + self.assertTypedEquals( + -2 ** 200 // 15, + F(-2**100, 3) // F(5, 2**100) + ) + self.assertTypedEquals( + 1, + F(5, 2**100) // F(3, 2**100) + ) + self.assertTypedEquals( + (1, F(2, 2**100)), + divmod(F(5, 2**100), F(3, 2**100)) + ) + self.assertTypedTupleEquals( + (-2 ** 200 // 15, + F(7, 1901475900342344102245054808064)), + divmod(F(-2**100, 3), F(5, 2**100)) + ) + + def testMixedArithmetic(self): + self.assertTypedEquals(F(11, 10), F(1, 10) + 1) + self.assertTypedEquals(1.1, F(1, 10) + 1.0) + self.assertTypedEquals(1.1 + 0j, F(1, 10) + (1.0 + 0j)) + self.assertTypedEquals(F(11, 10), 1 + F(1, 10)) + self.assertTypedEquals(1.1, 1.0 + F(1, 10)) + self.assertTypedEquals(1.1 + 0j, (1.0 + 0j) + F(1, 10)) + + self.assertTypedEquals(F(-9, 10), F(1, 10) - 1) + self.assertTypedEquals(-0.9, F(1, 10) - 1.0) + self.assertTypedEquals(-0.9 + 0j, F(1, 10) - (1.0 + 0j)) + self.assertTypedEquals(F(9, 10), 1 - F(1, 10)) + self.assertTypedEquals(0.9, 1.0 - F(1, 10)) + self.assertTypedEquals(0.9 + 0j, (1.0 + 0j) - F(1, 10)) + + self.assertTypedEquals(F(1, 10), F(1, 10) * 1) + self.assertTypedEquals(0.1, F(1, 10) * 1.0) + self.assertTypedEquals(0.1 + 0j, F(1, 10) * (1.0 + 0j)) + self.assertTypedEquals(F(1, 10), 1 * F(1, 10)) + self.assertTypedEquals(0.1, 1.0 * F(1, 10)) + self.assertTypedEquals(0.1 + 0j, (1.0 + 0j) * F(1, 10)) + + self.assertTypedEquals(F(1, 10), F(1, 10) / 1) + self.assertTypedEquals(0.1, F(1, 10) / 1.0) + self.assertTypedEquals(0.1 + 0j, F(1, 10) / (1.0 + 0j)) + self.assertTypedEquals(F(10, 1), 1 / F(1, 10)) + self.assertTypedEquals(10.0, 1.0 / F(1, 10)) + self.assertTypedEquals(10.0 + 0j, (1.0 + 0j) / F(1, 10)) + + self.assertTypedEquals(0, F(1, 10) // 1) + self.assertTypedEquals(0.0, F(1, 10) // 1.0) + self.assertTypedEquals(10, 1 // F(1, 10)) + self.assertTypedEquals(10**23, 10**22 // F(1, 10)) + self.assertTypedEquals(1.0 // 0.1, 1.0 // F(1, 10)) + + self.assertTypedEquals(F(1, 10), F(1, 10) % 1) + self.assertTypedEquals(0.1, F(1, 10) % 1.0) + self.assertTypedEquals(F(0, 1), 1 % F(1, 10)) + self.assertTypedEquals(1.0 % 0.1, 1.0 % F(1, 10)) + self.assertTypedEquals(0.1, F(1, 10) % float('inf')) + self.assertTypedEquals(float('-inf'), F(1, 10) % float('-inf')) + self.assertTypedEquals(float('inf'), F(-1, 10) % float('inf')) + self.assertTypedEquals(-0.1, F(-1, 10) % float('-inf')) + + self.assertTypedTupleEquals((0, F(1, 10)), divmod(F(1, 10), 1)) + self.assertTypedTupleEquals(divmod(0.1, 1.0), divmod(F(1, 10), 1.0)) + self.assertTypedTupleEquals((10, F(0)), divmod(1, F(1, 10))) + self.assertTypedTupleEquals(divmod(1.0, 0.1), divmod(1.0, F(1, 10))) + self.assertTypedTupleEquals(divmod(0.1, float('inf')), divmod(F(1, 10), float('inf'))) + self.assertTypedTupleEquals(divmod(0.1, float('-inf')), divmod(F(1, 10), float('-inf'))) + self.assertTypedTupleEquals(divmod(-0.1, float('inf')), divmod(F(-1, 10), float('inf'))) + self.assertTypedTupleEquals(divmod(-0.1, float('-inf')), divmod(F(-1, 10), float('-inf'))) + + # ** has more interesting conversion rules. + self.assertTypedEquals(F(100, 1), F(1, 10) ** -2) + self.assertTypedEquals(F(100, 1), F(10, 1) ** 2) + self.assertTypedEquals(0.1, F(1, 10) ** 1.0) + self.assertTypedEquals(0.1 + 0j, F(1, 10) ** (1.0 + 0j)) + self.assertTypedEquals(4 , 2 ** F(2, 1)) + z = pow(-1, F(1, 2)) + self.assertAlmostEqual(0, z.real) + self.assertEqual(1, z.imag) + self.assertTypedEquals(F(1, 4) , 2 ** F(-2, 1)) + self.assertTypedEquals(2.0 , 4 ** F(1, 2)) + self.assertTypedEquals(0.25, 2.0 ** F(-2, 1)) + self.assertTypedEquals(1.0 + 0j, (1.0 + 0j) ** F(1, 10)) + self.assertRaises(ZeroDivisionError, operator.pow, + F(0, 1), -2) + + @skip_skulpt + def testMixingWithDecimal(self): + # Decimal refuses mixed arithmetic (but not mixed comparisons) + self.assertRaises(TypeError, operator.add, + F(3,11), Decimal('3.1415926')) + self.assertRaises(TypeError, operator.add, + Decimal('3.1415926'), F(3,11)) + + def testComparisons(self): + self.assertTrue(F(1, 2) < F(2, 3)) + self.assertFalse(F(1, 2) < F(1, 2)) + self.assertTrue(F(1, 2) <= F(2, 3)) + self.assertTrue(F(1, 2) <= F(1, 2)) + self.assertFalse(F(2, 3) <= F(1, 2)) + self.assertTrue(F(1, 2) == F(1, 2)) + self.assertFalse(F(1, 2) == F(1, 3)) + self.assertFalse(F(1, 2) != F(1, 2)) + self.assertTrue(F(1, 2) != F(1, 3)) + + def testComparisonsDummyRational(self): + self.assertTrue(F(1, 2) == DummyRational(1, 2)) + self.assertTrue(DummyRational(1, 2) == F(1, 2)) + self.assertFalse(F(1, 2) == DummyRational(3, 4)) + self.assertFalse(DummyRational(3, 4) == F(1, 2)) + + self.assertTrue(F(1, 2) < DummyRational(3, 4)) + self.assertFalse(F(1, 2) < DummyRational(1, 2)) + self.assertFalse(F(1, 2) < DummyRational(1, 7)) + self.assertFalse(F(1, 2) > DummyRational(3, 4)) + self.assertFalse(F(1, 2) > DummyRational(1, 2)) + self.assertTrue(F(1, 2) > DummyRational(1, 7)) + self.assertTrue(F(1, 2) <= DummyRational(3, 4)) + self.assertTrue(F(1, 2) <= DummyRational(1, 2)) + self.assertFalse(F(1, 2) <= DummyRational(1, 7)) + self.assertFalse(F(1, 2) >= DummyRational(3, 4)) + self.assertTrue(F(1, 2) >= DummyRational(1, 2)) + self.assertTrue(F(1, 2) >= DummyRational(1, 7)) + + self.assertTrue(DummyRational(1, 2) < F(3, 4)) + self.assertFalse(DummyRational(1, 2) < F(1, 2)) + self.assertFalse(DummyRational(1, 2) < F(1, 7)) + self.assertFalse(DummyRational(1, 2) > F(3, 4)) + self.assertFalse(DummyRational(1, 2) > F(1, 2)) + self.assertTrue(DummyRational(1, 2) > F(1, 7)) + self.assertTrue(DummyRational(1, 2) <= F(3, 4)) + self.assertTrue(DummyRational(1, 2) <= F(1, 2)) + self.assertFalse(DummyRational(1, 2) <= F(1, 7)) + self.assertFalse(DummyRational(1, 2) >= F(3, 4)) + self.assertTrue(DummyRational(1, 2) >= F(1, 2)) + self.assertTrue(DummyRational(1, 2) >= F(1, 7)) + + def testComparisonsDummyFloat(self): + x = DummyFloat(1./3.) + y = F(1, 3) + self.assertTrue(x != y) + self.assertTrue(x < y or x > y) + self.assertFalse(x == y) + self.assertFalse(x <= y and x >= y) + self.assertTrue(y != x) + self.assertTrue(y < x or y > x) + self.assertFalse(y == x) + self.assertFalse(y <= x and y >= x) + + def testMixedLess(self): + self.assertTrue(2 < F(5, 2)) + self.assertFalse(2 < F(4, 2)) + self.assertTrue(F(5, 2) < 3) + self.assertFalse(F(4, 2) < 2) + + self.assertTrue(F(1, 2) < 0.6) + self.assertFalse(F(1, 2) < 0.4) + self.assertTrue(0.4 < F(1, 2)) + self.assertFalse(0.5 < F(1, 2)) + + self.assertFalse(float('inf') < F(1, 2)) + self.assertTrue(float('-inf') < F(0, 10)) + self.assertFalse(float('nan') < F(-3, 7)) + self.assertTrue(F(1, 2) < float('inf')) + self.assertFalse(F(17, 12) < float('-inf')) + self.assertFalse(F(144, -89) < float('nan')) + + def testMixedLessEqual(self): + self.assertTrue(0.5 <= F(1, 2)) + self.assertFalse(0.6 <= F(1, 2)) + self.assertTrue(F(1, 2) <= 0.5) + self.assertFalse(F(1, 2) <= 0.4) + self.assertTrue(2 <= F(4, 2)) + self.assertFalse(2 <= F(3, 2)) + self.assertTrue(F(4, 2) <= 2) + self.assertFalse(F(5, 2) <= 2) + + self.assertFalse(float('inf') <= F(1, 2)) + self.assertTrue(float('-inf') <= F(0, 10)) + self.assertFalse(float('nan') <= F(-3, 7)) + self.assertTrue(F(1, 2) <= float('inf')) + self.assertFalse(F(17, 12) <= float('-inf')) + self.assertFalse(F(144, -89) <= float('nan')) + + def testBigFloatComparisons(self): + # Because 10**23 can't be represented exactly as a float: + self.assertFalse(F(10**23) == float(10**23)) + # The first test demonstrates why these are important. + self.assertFalse(1e23 < float(F(math.trunc(1e23) + 1))) + self.assertTrue(1e23 < F(math.trunc(1e23) + 1)) + self.assertFalse(1e23 <= F(math.trunc(1e23) - 1)) + self.assertTrue(1e23 > F(math.trunc(1e23) - 1)) + self.assertFalse(1e23 >= F(math.trunc(1e23) + 1)) + + def testBigComplexComparisons(self): + self.assertFalse(F(10**23) == complex(10**23)) + self.assertRaises(TypeError, operator.gt, F(10**23), complex(10**23)) + self.assertRaises(TypeError, operator.le, F(10**23), complex(10**23)) + + x = F(3, 8) + z = complex(0.375, 0.0) + w = complex(0.375, 0.2) + self.assertTrue(x == z) + self.assertFalse(x != z) + self.assertFalse(x == w) + self.assertTrue(x != w) + for op in operator.lt, operator.le, operator.gt, operator.ge: + self.assertRaises(TypeError, op, x, z) + self.assertRaises(TypeError, op, z, x) + self.assertRaises(TypeError, op, x, w) + self.assertRaises(TypeError, op, w, x) + + def testMixedEqual(self): + self.assertTrue(0.5 == F(1, 2)) + self.assertFalse(0.6 == F(1, 2)) + self.assertTrue(F(1, 2) == 0.5) + self.assertFalse(F(1, 2) == 0.4) + self.assertTrue(2 == F(4, 2)) + self.assertFalse(2 == F(3, 2)) + self.assertTrue(F(4, 2) == 2) + self.assertFalse(F(5, 2) == 2) + self.assertFalse(F(5, 2) == float('nan')) + self.assertFalse(float('nan') == F(3, 7)) + self.assertFalse(F(5, 2) == float('inf')) + self.assertFalse(float('-inf') == F(2, 5)) + + def testStringification(self): + self.assertEqual("Fraction(7, 3)", repr(F(7, 3))) + self.assertEqual("Fraction(6283185307, 2000000000)", + repr(F('3.1415926535'))) + self.assertEqual("Fraction(-1, 100000000000000000000)", + repr(F(1, -10**20))) + self.assertEqual("7/3", str(F(7, 3))) + self.assertEqual("7", str(F(7, 1))) + + def testHash(self): + hmod = sys.hash_info.modulus + hinf = sys.hash_info.inf + self.assertEqual(hash(2.5), hash(F(5, 2))) + self.assertEqual(hash(10**50), hash(F(10**50))) + self.assertNotEqual(hash(float(10**23)), hash(F(10**23))) + self.assertEqual(hinf, hash(F(1, hmod))) + # Check that __hash__ produces the same value as hash(), for + # consistency with int and Decimal. (See issue #10356.) + self.assertEqual(hash(F(-1)), F(-1).__hash__()) + + def testApproximatePi(self): + # Algorithm borrowed from + # http://docs.python.org/lib/decimal-recipes.html + three = F(3) + lasts, t, s, n, na, d, da = 0, three, 3, 1, 0, 0, 24 + while abs(s - lasts) > F(1, 10**9): + lasts = s + n, na = n+na, na+8 + d, da = d+da, da+32 + t = (t * n) / d + s += t + self.assertAlmostEqual(math.pi, s) + + def testApproximateCos1(self): + # Algorithm borrowed from + # http://docs.python.org/lib/decimal-recipes.html + x = F(1) + i, lasts, s, fact, num, sign = 0, 0, F(1), 1, 1, 1 + while abs(s - lasts) > F(1, 10**9): + lasts = s + i += 2 + fact *= i * (i-1) + num *= x * x + sign *= -1 + s += num / fact * sign + self.assertAlmostEqual(math.cos(1), s) + + def test_copy_deepcopy_pickle(self): + r = F(13, 7) + dr = DummyFraction(13, 7) + # self.assertEqual(r, loads(dumps(r))) + self.assertEqual(id(r), id(copy(r))) + self.assertEqual(id(r), id(deepcopy(r))) + self.assertNotEqual(id(dr), id(copy(dr))) + self.assertNotEqual(id(dr), id(deepcopy(dr))) + self.assertTypedEquals(dr, copy(dr)) + self.assertTypedEquals(dr, deepcopy(dr)) + + def test_slots(self): + # Issue 4998 + r = F(13, 7) + self.assertRaises(AttributeError, setattr, r, 'a', 10) + + def test_int_subclass(self): + class myint(int): + def __mul__(self, other): + return type(self)(int(self) * int(other)) + def __floordiv__(self, other): + return type(self)(int(self) // int(other)) + def __mod__(self, other): + x = type(self)(int(self) % int(other)) + return x + @property + def numerator(self): + return type(self)(int(self)) + @property + def denominator(self): + return type(self)(1) + + f = fractions.Fraction(myint(1 * 3), myint(2 * 3)) + self.assertEqual(f.numerator, 1) + self.assertEqual(f.denominator, 2) + self.assertEqual(type(f.numerator), myint) + self.assertEqual(type(f.denominator), myint) + + +if __name__ == '__main__': + unittest.main() From 5524d3589ed81736465d8930fbaedc8d45f39c18 Mon Sep 17 00:00:00 2001 From: stu Date: Wed, 23 Nov 2022 14:19:30 +0800 Subject: [PATCH 098/137] fractions: finish it of which involves implementing float hashing function --- src/builtin/sys.js | 5 ++- src/float.js | 50 ++++++++++++++++++-------- src/int.js | 19 ++++++++-- src/lib/fractions.js | 86 +++++++++++++++++++++++++++++++------------- 4 files changed, 115 insertions(+), 45 deletions(-) diff --git a/src/builtin/sys.js b/src/builtin/sys.js index 0586da87ad..05a88fb98c 100644 --- a/src/builtin/sys.js +++ b/src/builtin/sys.js @@ -124,10 +124,9 @@ var $builtinmodule = function (name) { const HashInfoType = Sk.builtin.make_structseq("sys", "hash_info", hash_info_fields); + // not the same as CPYTHON - we want modulo to be a prime < Number.MAX_SAFE_INTEGER sys.hash_info = new HashInfoType( - [64, JSBI.BigInt("2305843009213693951"), 314159, 0, 1000003, "siphash24", 64, 128, 0].map((x) => - Sk.ffi.remapToPy(x) - ) + [32, 536870911, 314159, 0, 1000003, "siphash24", 32, 128, 0].map((x) => Sk.ffi.remapToPy(x)) ); diff --git a/src/float.js b/src/float.js index 483e7092c2..36f7d3227e 100644 --- a/src/float.js +++ b/src/float.js @@ -1,10 +1,10 @@ /** @typedef {Sk.builtin.object} */ var pyObject; -const hashMap = Object.create(null, { - Infinity: { value: 314159 }, - "-Infinity": { value: -314159 }, - NaN: { value: 0 }, -}); +// should match sys.hash_info +const _HASH_INF = 314159; +const _HASH_NAN = 0; +const _HASH_BITS = 29; +const _HASH_MOD = (1 << 29) - 1; /** * @constructor @@ -36,16 +36,38 @@ Sk.builtin.float_ = Sk.abstr.buildNativeClass("float", { tp$doc: "Convert a string or number to a floating point number, if possible.", tp$hash() { const v = this.v; - let hash = hashMap[v]; - if (hash !== undefined) { - return hash; - } else if (Number.isInteger(v)) { - hash = this.nb$int().tp$hash(); - } else { - hash = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER - Number.MAX_SAFE_INTEGER / 2); + if (!Number.isFinite(v)) { + if (Number.isNaN(v)) { + return _HASH_NAN; + } + return v > 0 ? _HASH_INF : -_HASH_INF; + } + let [m, e] = frexp(v); + let sign = 1; + if (m < 0) { + sign = -1; + m = -m; + } + + let x = 0; + while (m) { + x = ((x << 28) & _HASH_MOD) | x >> (_HASH_BITS - 28); + m *= 268435456; // 2 ** 28 + e -= 28; + y = Math.trunc(m); // pull out integer part + m -= y; + x += y; + if (x >= _HASH_MOD) { + x -= _HASH_MOD; + } + } + e = e >= 0 ? e % _HASH_BITS : _HASH_BITS-1-((-1-e) % _HASH_BITS); + x = ((x << e) & _HASH_MOD) | (x >> (_HASH_BITS - e)); + x *= sign; + if (x === -1) { + return -2; } - hashMap[this.v] = hash; - return hash; + return x; }, $r() { return new Sk.builtin.str(this.str$(10, true)); diff --git a/src/int.js b/src/int.js index 369dc151e6..61cae04e83 100644 --- a/src/int.js +++ b/src/int.js @@ -41,8 +41,18 @@ Sk.builtin.int_ = Sk.abstr.buildNativeClass("int", { return new Sk.builtin.str(this.v.toString()); }, tp$hash() { - const v = this.v; - return typeof v === "number" ? v : JSBI.toNumber(JSBI.remainder(v, JSBI.__MAX_SAFE)); + let v = this.v; + if (typeof v === "number") { + if (v === -1) { + return -2; + } + if (v < NUM_HASH_MOD && v > NEG_NUM_HASH_MOD) { + return v; + } + v = bigUp(v); + } + const rv = JSBI.toNumber(JSBI.remainder(v, BIG_HASH_MODULUS)); + return rv === -1 ? -2 : rv; }, tp$new(args, kwargs) { let x, base; @@ -198,7 +208,7 @@ Sk.builtin.int_ = Sk.abstr.buildNativeClass("int", { if (ret !== undefined) { return ret.nb$remainder(mod); } - return new Sk.builtin.int_(JSBI.powermod(bigUp(v), bigUp(w), bigUp(mod.v))); + return new Sk.builtin.int_(JSBI.numberIfSafe(JSBI.powermod(bigUp(v), bigUp(w), bigUp(mod.v)))); } // if we're here then we've fallen through so do bigint exponentiate return new Sk.builtin.int_(JSBI.exponentiate(bigUp(v), bigUp(w))); @@ -522,6 +532,9 @@ function cloneSelf() { return new Sk.builtin.int_(this.v); } +const NUM_HASH_MOD = 536870911; +const NEG_NUM_HASH_MOD = -536870911; +const BIG_HASH_MODULUS = JSBI.BigInt("536870911"); const DBL_MANT_DIG = Math.log2(Number.MAX_SAFE_INTEGER); const DBL_MAX_EXP = JSBI.BigInt(Math.floor(Math.log2(Number.MAX_VALUE))); const DBL_MIN_EXP = Math.ceil(Math.log2(Number.MIN_VALUE)); diff --git a/src/lib/fractions.js b/src/lib/fractions.js index 41d9d5b811..8a01557e82 100644 --- a/src/lib/fractions.js +++ b/src/lib/fractions.js @@ -15,11 +15,11 @@ function $builtinmodule(name) { ); } -function fractionsMod({math, sys}) { +function fractionsMod({ math, sys }) { const { builtin: { int_: pyInt, - bool: { true$: pyTrue }, + bool: { true$: pyTrue, false$: pyFalse }, none: { none$: pyNone }, NotImplemented: { NotImplemented$: pyNotImplemented }, tuple: pyTuple, @@ -33,7 +33,7 @@ function fractionsMod({math, sys}) { NotImplementedError: pyNotImplementedError, abs: pyAbs, round: pyRound, - power: pyPower, + pow: pyPower, }, ffi: { remapToPy: toPy }, abstr: { buildNativeClass, copyKeywordsToNamedArgs, numberBinOp, typeName, lookupSpecial, checkArgsLen }, @@ -48,8 +48,6 @@ function fractionsMod({math, sys}) { const _1 = new pyInt(1); const _2 = new pyInt(2); const _10 = new pyInt(10); - const _neg_1 = new pyInt(-1); - const _neg_2 = new pyInt(-2); const s_numerator = new pyStr("numerator"); const s_denominator = new pyStr("denominator"); const s_int_ratio = new pyStr("as_integer_ratio"); @@ -81,7 +79,7 @@ function fractionsMod({math, sys}) { function _operator_fallbacks(monomorphic, fallback) { const forward = function (other) { - if (isTrue(pyIsInstance(other, _NUMBERS_RATIONAL))) { + if (isRational(other)) { return monomorphic(this, other); } else if (other instanceof pyFloat) { return fallback(this.nb$float(), other); @@ -92,7 +90,7 @@ function fractionsMod({math, sys}) { } }; const reverse = function (other) { - if (isTrue(pyIsInstance(other, _NUMBERS_RATIONAL))) { + if (isRational(other)) { return monomorphic(other, this); } else if (other instanceof pyFloat) { return fallback(other, this.nb$float()); @@ -170,7 +168,7 @@ function fractionsMod({math, sys}) { self.$num = numerator; self.$den = _1; return self; - } else if (isTrue(pyIsInstance(numerator, _NUMBERS_RATIONAL))) { + } else if (isRational(numerator)) { self.$num = getNumer(numerator); self.$den = getDenom(numerator); return self; @@ -214,10 +212,7 @@ function fractionsMod({math, sys}) { } } else if (numerator.ob$type === pyInt && denominator.ob$type === pyInt) { // normal case pass - } else if ( - isTrue(pyIsInstance(numerator, _NUMBERS_RATIONAL)) && - isTrue(pyIsInstance(denominator, _NUMBERS_RATIONAL)) - ) { + } else if (isRational(numerator) && isRational(denominator)) { [numerator, denominator] = [ mul(getNumer(numerator), getDenom(denominator)), mul(getNumer(denominator), getDenom(numerator)), @@ -253,15 +248,16 @@ function fractionsMod({math, sys}) { return new pyStr(`${this.$num}/${this.$den}`); }, tp$hash() { - const dinv = pyPower(this.$den, sub(_PyHASH_MODULUS, 2), _PyHASH_MODULUS); + const dinv = pyPower(this.$den, sub(_PyHASH_MODULUS, _2), _PyHASH_MODULUS); let hash_; if (!isTrue(dinv)) { hash_ = _PyHASH_INF; } else { hash_ = mod(mul(pyAbs(this.$num), dinv), _PyHASH_MODULUS); } - const rv = ge(this, _0) ? hash_ : hash_.nb$negative(); - return eq(rv, _neg_1) ? _neg_2 : rv; + let rv = ge(this, _0) ? hash_ : hash_.nb$negative(); + rv = rv.tp$hash(); + return rv === -1 ? -2 : rv; }, tp$richcompare(other, OP) { const op = (a, b) => richCompareBool(a, b, OP); @@ -282,7 +278,7 @@ function fractionsMod({math, sys}) { } } - if (isTrue(pyIsInstance(other, _NUMBERS_RATIONAL))) { + if (isRational(other)) { return op(mul(getNumer(this), getDenom(other)), mul(getDenom(this), getNumer(other))); } if (other instanceof pyFloat) { @@ -311,20 +307,29 @@ function fractionsMod({math, sys}) { nb$remainder, nb$reflected_remainder, nb$power(other) { - if (isTrue(pyIsInstance(other, _NUMBERS_RATIONAL))) { + if (isRational(other)) { if (eq(getDenom(other), _1)) { let power = getNumer(other); if (ge(power, _0)) { - return pyCall(Fraction, [pow(this.$num, power), pow(this.$den, power)]); + return pyCall( + Fraction, + [pow(this.$num, power), pow(this.$den, power)], + ["_normalize", pyFalse] + ); } else if (ge(this.$num, _0)) { power = power.nb$negative(); - return pyCall(Fraction, [pow(this.$den, power), pow(this.$num, power)]); + return pyCall( + Fraction, + [pow(this.$den, power), pow(this.$num, power)], + ["_normalize", pyFalse] + ); } else { power = power.nb$negative(); - return pyCall(Fraction, [ - pow(this.$den.nb$negative(), power), - pow(this.$num.nb$negative(), power), - ]); + return pyCall( + Fraction, + [pow(this.$den.nb$negative(), power), pow(this.$num.nb$negative(), power)], + ["_normalize", pyFalse] + ); } } else { return pow(this.nb$float(), pyCall(pyFloat, [other])); @@ -338,7 +343,7 @@ function fractionsMod({math, sys}) { return pow(other, this.$num); } - if (isTrue(pyIsInstance(other, _NUMBERS_RATIONAL))) { + if (isRational(other)) { return pow(new Fraction(getNumer(other), getDenom(other)), this); } @@ -371,7 +376,37 @@ function fractionsMod({math, sys}) { }, $flags: METH_NO_ARGS, }, - limit_denominator: {}, + limit_denominator: { + $meth(max_denominator) { + if (lt(max_denominator, _1)) { + throw new pyValueError("max_denominator should be at least 1"); + } + if (ge(max_denominator, this.$den)) { + return pyCall(Fraction, [this]); + } + let [p0, q0, p1, q1] = [_0, _1, _1, _0]; + let n = this.$num; + let d = this.$den; + while (true) { + const a = floorDiv(n, d); + const q2 = add(q0, mul(a, q1)); + if (lt(max_denominator, q2)) { + break; + } + [p0, q0, p1, q1] = [p1, q1, add(p0, mul(a, p1)), q2]; + [n, d] = [d, sub(n, mul(a, d))]; + } + const k = floorDiv(sub(max_denominator, q0), q1); + const b1 = pyCall(Fraction, [add(p0, mul(k, p1)), add(q0, mul(k, q1))]); + const b2 = pyCall(Fraction, [p1, q1]); + if (ge(pyAbs(sub(b1, this)), pyAbs(sub(b2, this)))) { + return b2; + } else { + return b1; + } + }, + $flags: { NamedArgs: ["max_denominator"], Defaults: [new pyInt(1000000)] }, + }, __trunc__: { $meth() { if (lt(this.$num, _0)) { @@ -499,6 +534,7 @@ function fractionsMod({math, sys}) { })); const _NUMBERS_RATIONAL = new pyTuple([pyInt, Fraction]); + const isRational = (obj) => isTrue(pyIsInstance(obj, _NUMBERS_RATIONAL)); return fractionsMod; } From e662563ebf3e91d4b7e5a2e8b855236fe6ef2ca4 Mon Sep 17 00:00:00 2001 From: stu Date: Wed, 23 Nov 2022 14:28:26 +0800 Subject: [PATCH 099/137] Fix some failing tests after changes --- src/lib/math.js | 6 +++--- test/run/t498.py | 10 +++++----- test/run/t498.py.real | 1 - 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/lib/math.js b/src/lib/math.js index 1abbf0f5b6..407a68da42 100644 --- a/src/lib/math.js +++ b/src/lib/math.js @@ -1,7 +1,7 @@ const $builtinmodule = function (name) { const { - builtin: { str: pyStr, float_: pyFloat, TypeError: pyTypeError, pyCheckType, checkNumber }, + builtin: { str: pyStr, int_: pyInt, float_: pyFloat, TypeError: pyTypeError, pyCheckType, checkNumber }, abstr: { lookupSpecial }, misceval: { callsimOrSuspendArray: pyCallOrSuspend }, } = Sk; @@ -30,7 +30,7 @@ const $builtinmodule = function (name) { } else { _x = x.v; } - return new pyFloat(Math.ceil(_x)); + return new pyInt(Math.ceil(_x)); }; function comb(n, k) { @@ -149,7 +149,7 @@ const $builtinmodule = function (name) { pyCheckType("x", "number", checkNumber(x)); _x = Sk.builtin.asnum$(x); } - return new pyFloat(Math.floor(_x)); + return new pyInt(Math.floor(_x)); }; function fmod(x, y) { diff --git a/test/run/t498.py b/test/run/t498.py index b4ce38081b..b69ff70d97 100644 --- a/test/run/t498.py +++ b/test/run/t498.py @@ -61,11 +61,11 @@ except TypeError as e: print repr(e) -try: - print pow(-2.5, 3.7) - print "you shouldn't see this" -except ValueError as e: - print repr(e) +# try: +# print pow(-2.5, 3.7) +# print "you shouldn't see this" +# except ValueError as e: +# print repr(e) try: print pow(4.0, 5.0, 3) diff --git a/test/run/t498.py.real b/test/run/t498.py.real index 208d058cc3..8050bca806 100644 --- a/test/run/t498.py.real +++ b/test/run/t498.py.real @@ -52,6 +52,5 @@ floating point and long integers ERROR CHECKING: TypeError("unsupported operand type(s) for ** or pow(): 'list' and 'str'") TypeError("unsupported operand type(s) for ** or pow(): 'list', 'str', 'int'") -ValueError('negative number cannot be raised to a fractional power') TypeError('pow() 3rd argument not allowed unless all arguments are integers') ValueError('pow() 2nd argument cannot be negative when 3rd argument specified') From 174f10122b3b6c564ee449aa9405fe2d19d21326 Mon Sep 17 00:00:00 2001 From: stu Date: Wed, 23 Nov 2022 17:01:42 +0800 Subject: [PATCH 100/137] Add some hash tests from cpython, plus fraction.js comments --- src/lib/fractions.js | 2 +- test/unit3/test_builtin.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/lib/fractions.js b/src/lib/fractions.js index 8a01557e82..1d981c7ddf 100644 --- a/src/lib/fractions.js +++ b/src/lib/fractions.js @@ -1,4 +1,3 @@ -// @ts-check function $builtinmodule(name) { const mods = {}; @@ -514,6 +513,7 @@ function fractionsMod({ math, sys }) { return this.$den; }, }, + // seems silly really but it gets tested _numerator: { $get() { return this.$num; diff --git a/test/unit3/test_builtin.py b/test/unit3/test_builtin.py index bf4c8cac5a..1913d25d3c 100644 --- a/test/unit3/test_builtin.py +++ b/test/unit3/test_builtin.py @@ -722,6 +722,27 @@ def __init__(self): self.assertTrue(flag2) self.assertFalse(hasattr(F,'a')) + def test_hash(self): + hash(None) + self.assertEqual(hash(1), hash(1)) + self.assertEqual(hash(1), hash(1.0)) + hash('spam') + self.assertEqual(hash('spam'), hash(b'spam')) + hash((0,1,2,3)) + def f(): pass + hash(f) + self.assertRaises(TypeError, hash, []) + self.assertRaises(TypeError, hash, {}) + # Bug 1536021: Allow hash to return long objects + class X: + def __hash__(self): + return 2**100 + self.assertEqual(type(hash(X())), int) + class Z(int): + def __hash__(self): + return self + self.assertEqual(hash(Z(42)), hash(42)) + def test_hex(self): self.assertEqual(hex(16), '0x10') self.assertEqual(hex(-16), '-0x10') From 9e5aefb2f67e22a5075a857921c94da401b3e6e4 Mon Sep 17 00:00:00 2001 From: stu Date: Wed, 23 Nov 2022 17:06:10 +0800 Subject: [PATCH 101/137] fix build issues --- src/float.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/float.js b/src/float.js index 36f7d3227e..7596df08d3 100644 --- a/src/float.js +++ b/src/float.js @@ -49,7 +49,7 @@ Sk.builtin.float_ = Sk.abstr.buildNativeClass("float", { m = -m; } - let x = 0; + let x = 0, y; while (m) { x = ((x << 28) & _HASH_MOD) | x >> (_HASH_BITS - 28); m *= 268435456; // 2 ** 28 From 4bffd400b57c5d0cf577a160d3bcf4e03e1adcb5 Mon Sep 17 00:00:00 2001 From: stu Date: Wed, 23 Nov 2022 17:12:23 +0800 Subject: [PATCH 102/137] Fix bug in float.js --- src/float.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/float.js b/src/float.js index 7596df08d3..2d26b86b06 100644 --- a/src/float.js +++ b/src/float.js @@ -474,7 +474,7 @@ function remainder(v, w) { return new Sk.builtin.float_(0); } if (w === Infinity) { - if (v === Infinity || this.v === -Infinity) { + if (v === Infinity || v === -Infinity) { return new Sk.builtin.float_(NaN); } else if (v > 0) { return new Sk.builtin.float_(v); From a5d44e874ae3a472cda8b24297f0753ef045b3a7 Mon Sep 17 00:00:00 2001 From: stu Date: Sat, 3 Dec 2022 23:00:25 +0800 Subject: [PATCH 103/137] fix not possible for null to be an attribute --- gen/parse_tables.js | 4 ++-- src/pgen/parser/grammar.py | 2 +- test/unit3/test_grammar.py | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/gen/parse_tables.js b/gen/parse_tables.js index 3767a24449..29b2a327a4 100644 --- a/gen/parse_tables.js +++ b/gen/parse_tables.js @@ -1546,7 +1546,7 @@ labels: [1, 'True'], [3, null], [1, 'not'], - [1, 'null'], + [1, 'None'], [55, null], [2, null], [25, null], @@ -1714,7 +1714,7 @@ labels: [341, null]], keywords: {'False': 33, - 'null': 9, + 'None': 9, 'True': 6, 'and': 47, 'as': 108, diff --git a/src/pgen/parser/grammar.py b/src/pgen/parser/grammar.py index bb2617d374..049d420e65 100644 --- a/src/pgen/parser/grammar.py +++ b/src/pgen/parser/grammar.py @@ -142,7 +142,7 @@ def genjs(self): str(self.start) + "\n};\n" # ick, tuple -> list and None -> null - ).replace("(", "[").replace(")", "]").replace("None", "null") + ).replace("(", "[").replace(")", "]").replace("None]", "null]") # Map from operator to number (since tokenize doesn't do this) diff --git a/test/unit3/test_grammar.py b/test/unit3/test_grammar.py index e557741ccb..1f11aac035 100644 --- a/test/unit3/test_grammar.py +++ b/test/unit3/test_grammar.py @@ -628,10 +628,9 @@ def f(self, *, __kw: 1): class Ham(Spam): pass self.assertEqual(Spam.f.__annotations__, {'_Spam__kw': 1}) self.assertEqual(Ham.f.__annotations__, {'_Spam__kw': 1}) - # @TODO skulpt doesn't allow null # Check for SF Bug #1697248 - mixing decorators and a return annotation - def _null(x): return x - @_null + def null(x): return x + @null def f(x) -> list: pass self.assertEqual(f.__annotations__, {'return': list}) From 0bf37ad59b47117609281361b68163a9d8515c96 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Dec 2022 10:29:03 +0000 Subject: [PATCH 104/137] build(deps): bump decode-uri-component from 0.2.0 to 0.2.2 Bumps [decode-uri-component](https://github.com/SamVerschueren/decode-uri-component) from 0.2.0 to 0.2.2. - [Release notes](https://github.com/SamVerschueren/decode-uri-component/releases) - [Commits](https://github.com/SamVerschueren/decode-uri-component/compare/v0.2.0...v0.2.2) --- updated-dependencies: - dependency-name: decode-uri-component dependency-type: indirect ... Signed-off-by: dependabot[bot] --- package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index 09532e4fa2..e00d664613 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1715,9 +1715,9 @@ } }, "node_modules/decode-uri-component": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", - "integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=", + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.2.tgz", + "integrity": "sha512-FqUYQ+8o158GyGTrMFJms9qh3CqTKvAqgqsTnkLI8sKu0028orqBhxNMFkFen0zGyg6epACD32pjVk58ngIErQ==", "dev": true, "engines": { "node": ">=0.10" @@ -8450,9 +8450,9 @@ "dev": true }, "decode-uri-component": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", - "integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=", + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.2.tgz", + "integrity": "sha512-FqUYQ+8o158GyGTrMFJms9qh3CqTKvAqgqsTnkLI8sKu0028orqBhxNMFkFen0zGyg6epACD32pjVk58ngIErQ==", "dev": true }, "deep-is": { From 944f438d884b4039e1a0f585f31156c2b7a7fdae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Dec 2022 11:55:56 +0000 Subject: [PATCH 105/137] build(deps-dev): bump express from 4.17.1 to 4.17.3 Bumps [express](https://github.com/expressjs/express) from 4.17.1 to 4.17.3. - [Release notes](https://github.com/expressjs/express/releases) - [Changelog](https://github.com/expressjs/express/blob/master/History.md) - [Commits](https://github.com/expressjs/express/compare/4.17.1...4.17.3) --- updated-dependencies: - dependency-name: express dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- package-lock.json | 441 +++++++++++++++++++++++++--------------------- package.json | 2 +- 2 files changed, 243 insertions(+), 200 deletions(-) diff --git a/package-lock.json b/package-lock.json index e00d664613..813fcf5973 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,7 +22,7 @@ "ejs": "^3.1.7", "eslint": "^5.16.0", "eslint-loader": "^2.1.2", - "express": "^4.17.0", + "express": "^4.17.3", "fastestsmallesttextencoderdecoder": "^1.0.22", "git-revision-webpack-plugin": "^3.0.3", "google-closure-compiler": "^20210202.0.0", @@ -376,13 +376,13 @@ "dev": true }, "node_modules/accepts": { - "version": "1.3.7", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", - "integrity": "sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA==", + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", "dev": true, "dependencies": { - "mime-types": "~2.1.24", - "negotiator": "0.6.2" + "mime-types": "~2.1.34", + "negotiator": "0.6.3" }, "engines": { "node": ">= 0.6" @@ -780,21 +780,21 @@ "dev": true }, "node_modules/body-parser": { - "version": "1.19.0", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz", - "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==", + "version": "1.19.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.2.tgz", + "integrity": "sha512-SAAwOxgoCKMGs9uUAUFHygfLAyaniaoun6I8mFY9pRAJL9+Kec34aU+oIjDhTycub1jozEfEwx1W1IuOYxVSFw==", "dev": true, "dependencies": { - "bytes": "3.1.0", + "bytes": "3.1.2", "content-type": "~1.0.4", "debug": "2.6.9", "depd": "~1.1.2", - "http-errors": "1.7.2", + "http-errors": "1.8.1", "iconv-lite": "0.4.24", "on-finished": "~2.3.0", - "qs": "6.7.0", - "raw-body": "2.4.0", - "type-is": "~1.6.17" + "qs": "6.9.7", + "raw-body": "2.4.3", + "type-is": "~1.6.18" }, "engines": { "node": ">= 0.8" @@ -812,7 +812,7 @@ "node_modules/body-parser/node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", "dev": true }, "node_modules/brace-expansion": { @@ -987,9 +987,9 @@ "dev": true }, "node_modules/bytes": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", - "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", "dev": true, "engines": { "node": ">= 0.8" @@ -1461,17 +1461,37 @@ "dev": true }, "node_modules/content-disposition": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", - "integrity": "sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g==", + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", "dev": true, "dependencies": { - "safe-buffer": "5.1.2" + "safe-buffer": "5.2.1" }, "engines": { "node": ">= 0.6" } }, + "node_modules/content-disposition/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/content-type": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", @@ -1482,9 +1502,9 @@ } }, "node_modules/cookie": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz", - "integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg==", + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.2.tgz", + "integrity": "sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA==", "dev": true, "engines": { "node": ">= 0.6" @@ -1801,7 +1821,7 @@ "node_modules/depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=", + "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", "dev": true, "engines": { "node": ">= 0.6" @@ -1820,7 +1840,7 @@ "node_modules/destroy": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", - "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=", + "integrity": "sha512-3NdhDuEXnfun/z7x9GOElY49LoqVHoGScmOKwmxhsS8N5Y+Z8KyPPDnaSzqWgYt/ji4mqwfTS34Htrk0zPIXVg==", "dev": true }, "node_modules/detect-file": { @@ -2257,7 +2277,7 @@ "node_modules/etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", "dev": true, "engines": { "node": ">= 0.6" @@ -2352,17 +2372,17 @@ } }, "node_modules/express": { - "version": "4.17.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz", - "integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==", + "version": "4.17.3", + "resolved": "https://registry.npmjs.org/express/-/express-4.17.3.tgz", + "integrity": "sha512-yuSQpz5I+Ch7gFrPCk4/c+dIBKlQUxtgwqzph132bsT6qhuzss6I8cLJQz7B3rFblzd6wtcI0ZbGltH/C4LjUg==", "dev": true, "dependencies": { - "accepts": "~1.3.7", + "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.19.0", - "content-disposition": "0.5.3", + "body-parser": "1.19.2", + "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.4.0", + "cookie": "0.4.2", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "~1.1.2", @@ -2376,13 +2396,13 @@ "on-finished": "~2.3.0", "parseurl": "~1.3.3", "path-to-regexp": "0.1.7", - "proxy-addr": "~2.0.5", - "qs": "6.7.0", + "proxy-addr": "~2.0.7", + "qs": "6.9.7", "range-parser": "~1.2.1", - "safe-buffer": "5.1.2", - "send": "0.17.1", - "serve-static": "1.14.1", - "setprototypeof": "1.1.1", + "safe-buffer": "5.2.1", + "send": "0.17.2", + "serve-static": "1.14.2", + "setprototypeof": "1.2.0", "statuses": "~1.5.0", "type-is": "~1.6.18", "utils-merge": "1.0.1", @@ -2407,6 +2427,26 @@ "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", "dev": true }, + "node_modules/express/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/extend-shallow": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz", @@ -2764,9 +2804,9 @@ } }, "node_modules/forwarded": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", - "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=", + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", "dev": true, "engines": { "node": ">= 0.6" @@ -2787,7 +2827,7 @@ "node_modules/fresh": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", "dev": true, "engines": { "node": ">= 0.6" @@ -3193,27 +3233,21 @@ } }, "node_modules/http-errors": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", - "integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==", + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", + "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", "dev": true, "dependencies": { "depd": "~1.1.2", - "inherits": "2.0.3", - "setprototypeof": "1.1.1", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", "statuses": ">= 1.5.0 < 2", - "toidentifier": "1.0.0" + "toidentifier": "1.0.1" }, "engines": { "node": ">= 0.6" } }, - "node_modules/http-errors/node_modules/inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", - "dev": true - }, "node_modules/https-browserify": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz", @@ -4118,7 +4152,7 @@ "node_modules/media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", "dev": true, "engines": { "node": ">= 0.6" @@ -4211,21 +4245,21 @@ } }, "node_modules/mime-db": { - "version": "1.46.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.46.0.tgz", - "integrity": "sha512-svXaP8UQRZ5K7or+ZmfNhg2xX3yKDMUzqadsSqi4NCH/KomcH75MAMYAGVlvXn4+b/xOPhS3I2uHKRUzvjY7BQ==", + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", "dev": true, "engines": { "node": ">= 0.6" } }, "node_modules/mime-types": { - "version": "2.1.29", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.29.tgz", - "integrity": "sha512-Y/jMt/S5sR9OaqteJtslsFZKWOIIqMACsJSiHghlCAyhf7jfVYjKBmLiX8OgpWeW+fjJ2b+Az69aPFPkUOY6xQ==", + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", "dev": true, "dependencies": { - "mime-db": "1.46.0" + "mime-db": "1.52.0" }, "engines": { "node": ">= 0.6" @@ -4383,9 +4417,9 @@ "dev": true }, "node_modules/negotiator": { - "version": "0.6.2", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", - "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==", + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", "dev": true, "engines": { "node": ">= 0.6" @@ -4933,12 +4967,12 @@ "dev": true }, "node_modules/proxy-addr": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", - "integrity": "sha512-dh/frvCBVmSsDYzw6n926jv974gddhkFPfiN8hPOi30Wax25QZyZEGveluCgliBnqmuM+UJmBErbAUFIoDbjOw==", + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", "dev": true, "dependencies": { - "forwarded": "~0.1.2", + "forwarded": "0.2.0", "ipaddr.js": "1.9.1" }, "engines": { @@ -5018,12 +5052,15 @@ } }, "node_modules/qs": { - "version": "6.7.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", - "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==", + "version": "6.9.7", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.7.tgz", + "integrity": "sha512-IhMFgUmuNpyRfxA90umL7ByLlgRXu6tIfKPpF5TmcfRLlLCckfP/g3IQmju6jjpu+Hh8rA+2p6A27ZSPOOHdKw==", "dev": true, "engines": { "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/querystring": { @@ -5073,13 +5110,13 @@ } }, "node_modules/raw-body": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz", - "integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==", + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.3.tgz", + "integrity": "sha512-UlTNLIcu0uzb4D2f4WltY6cVjLi+/jEN4lgEUj3E04tpMDpUlkBo/eSn6zou9hum2VMNpCCUone0O0WeJim07g==", "dev": true, "dependencies": { - "bytes": "3.1.0", - "http-errors": "1.7.2", + "bytes": "3.1.2", + "http-errors": "1.8.1", "iconv-lite": "0.4.24", "unpipe": "1.0.0" }, @@ -5407,9 +5444,9 @@ } }, "node_modules/send": { - "version": "0.17.1", - "resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz", - "integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==", + "version": "0.17.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.17.2.tgz", + "integrity": "sha512-UJYB6wFSJE3G00nEivR5rgWp8c2xXvJ3OPWPhmuteU0IKj8nKbG3DrjiOmLwpnHGYWAVwA69zmTm++YG0Hmwww==", "dev": true, "dependencies": { "debug": "2.6.9", @@ -5419,9 +5456,9 @@ "escape-html": "~1.0.3", "etag": "~1.8.1", "fresh": "0.5.2", - "http-errors": "~1.7.2", + "http-errors": "1.8.1", "mime": "1.6.0", - "ms": "2.1.1", + "ms": "2.1.3", "on-finished": "~2.3.0", "range-parser": "~1.2.1", "statuses": "~1.5.0" @@ -5442,13 +5479,13 @@ "node_modules/send/node_modules/debug/node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", "dev": true }, "node_modules/send/node_modules/ms": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", - "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "dev": true }, "node_modules/serialize-javascript": { @@ -5458,15 +5495,15 @@ "dev": true }, "node_modules/serve-static": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz", - "integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==", + "version": "1.14.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.2.tgz", + "integrity": "sha512-+TMNA9AFxUEGuC0z2mevogSnn9MXKb4fa7ngeRMJaaGv8vTwnIEkKi+QGvPt33HSnf8pRS+WGM0EbMtCJLKMBQ==", "dev": true, "dependencies": { "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "parseurl": "~1.3.3", - "send": "0.17.1" + "send": "0.17.2" }, "engines": { "node": ">= 0.8.0" @@ -5512,9 +5549,9 @@ "dev": true }, "node_modules/setprototypeof": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", - "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", "dev": true }, "node_modules/sha.js": { @@ -6244,9 +6281,9 @@ } }, "node_modules/toidentifier": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", - "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", "dev": true, "engines": { "node": ">=0.6" @@ -7292,13 +7329,13 @@ "dev": true }, "accepts": { - "version": "1.3.7", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", - "integrity": "sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA==", + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", "dev": true, "requires": { - "mime-types": "~2.1.24", - "negotiator": "0.6.2" + "mime-types": "~2.1.34", + "negotiator": "0.6.3" } }, "acorn": { @@ -7623,21 +7660,21 @@ "dev": true }, "body-parser": { - "version": "1.19.0", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz", - "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==", + "version": "1.19.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.2.tgz", + "integrity": "sha512-SAAwOxgoCKMGs9uUAUFHygfLAyaniaoun6I8mFY9pRAJL9+Kec34aU+oIjDhTycub1jozEfEwx1W1IuOYxVSFw==", "dev": true, "requires": { - "bytes": "3.1.0", + "bytes": "3.1.2", "content-type": "~1.0.4", "debug": "2.6.9", "depd": "~1.1.2", - "http-errors": "1.7.2", + "http-errors": "1.8.1", "iconv-lite": "0.4.24", "on-finished": "~2.3.0", - "qs": "6.7.0", - "raw-body": "2.4.0", - "type-is": "~1.6.17" + "qs": "6.9.7", + "raw-body": "2.4.3", + "type-is": "~1.6.18" }, "dependencies": { "debug": { @@ -7652,7 +7689,7 @@ "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", "dev": true } } @@ -7824,9 +7861,9 @@ "dev": true }, "bytes": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", - "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", "dev": true }, "cacache": { @@ -8225,12 +8262,20 @@ "dev": true }, "content-disposition": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", - "integrity": "sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g==", + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", "dev": true, "requires": { - "safe-buffer": "5.1.2" + "safe-buffer": "5.2.1" + }, + "dependencies": { + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true + } } }, "content-type": { @@ -8240,9 +8285,9 @@ "dev": true }, "cookie": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz", - "integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg==", + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.2.tgz", + "integrity": "sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA==", "dev": true }, "cookie-signature": { @@ -8520,7 +8565,7 @@ "depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=", + "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", "dev": true }, "des.js": { @@ -8536,7 +8581,7 @@ "destroy": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", - "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=", + "integrity": "sha512-3NdhDuEXnfun/z7x9GOElY49LoqVHoGScmOKwmxhsS8N5Y+Z8KyPPDnaSzqWgYt/ji4mqwfTS34Htrk0zPIXVg==", "dev": true }, "detect-file": { @@ -8901,7 +8946,7 @@ "etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", "dev": true }, "events": { @@ -8980,17 +9025,17 @@ } }, "express": { - "version": "4.17.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz", - "integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==", + "version": "4.17.3", + "resolved": "https://registry.npmjs.org/express/-/express-4.17.3.tgz", + "integrity": "sha512-yuSQpz5I+Ch7gFrPCk4/c+dIBKlQUxtgwqzph132bsT6qhuzss6I8cLJQz7B3rFblzd6wtcI0ZbGltH/C4LjUg==", "dev": true, "requires": { - "accepts": "~1.3.7", + "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.19.0", - "content-disposition": "0.5.3", + "body-parser": "1.19.2", + "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.4.0", + "cookie": "0.4.2", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "~1.1.2", @@ -9004,13 +9049,13 @@ "on-finished": "~2.3.0", "parseurl": "~1.3.3", "path-to-regexp": "0.1.7", - "proxy-addr": "~2.0.5", - "qs": "6.7.0", + "proxy-addr": "~2.0.7", + "qs": "6.9.7", "range-parser": "~1.2.1", - "safe-buffer": "5.1.2", - "send": "0.17.1", - "serve-static": "1.14.1", - "setprototypeof": "1.1.1", + "safe-buffer": "5.2.1", + "send": "0.17.2", + "serve-static": "1.14.2", + "setprototypeof": "1.2.0", "statuses": "~1.5.0", "type-is": "~1.6.18", "utils-merge": "1.0.1", @@ -9031,6 +9076,12 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", "dev": true + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true } } }, @@ -9340,9 +9391,9 @@ "dev": true }, "forwarded": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", - "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=", + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", "dev": true }, "fragment-cache": { @@ -9357,7 +9408,7 @@ "fresh": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", "dev": true }, "from2": { @@ -9679,24 +9730,16 @@ } }, "http-errors": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", - "integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==", + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", + "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", "dev": true, "requires": { "depd": "~1.1.2", - "inherits": "2.0.3", - "setprototypeof": "1.1.1", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", "statuses": ">= 1.5.0 < 2", - "toidentifier": "1.0.0" - }, - "dependencies": { - "inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", - "dev": true - } + "toidentifier": "1.0.1" } }, "https-browserify": { @@ -10422,7 +10465,7 @@ "media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", "dev": true }, "memory-fs": { @@ -10499,18 +10542,18 @@ "dev": true }, "mime-db": { - "version": "1.46.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.46.0.tgz", - "integrity": "sha512-svXaP8UQRZ5K7or+ZmfNhg2xX3yKDMUzqadsSqi4NCH/KomcH75MAMYAGVlvXn4+b/xOPhS3I2uHKRUzvjY7BQ==", + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", "dev": true }, "mime-types": { - "version": "2.1.29", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.29.tgz", - "integrity": "sha512-Y/jMt/S5sR9OaqteJtslsFZKWOIIqMACsJSiHghlCAyhf7jfVYjKBmLiX8OgpWeW+fjJ2b+Az69aPFPkUOY6xQ==", + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", "dev": true, "requires": { - "mime-db": "1.46.0" + "mime-db": "1.52.0" } }, "mimic-fn": { @@ -10646,9 +10689,9 @@ "dev": true }, "negotiator": { - "version": "0.6.2", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", - "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==", + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", "dev": true }, "neo-async": { @@ -11085,12 +11128,12 @@ "dev": true }, "proxy-addr": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", - "integrity": "sha512-dh/frvCBVmSsDYzw6n926jv974gddhkFPfiN8hPOi30Wax25QZyZEGveluCgliBnqmuM+UJmBErbAUFIoDbjOw==", + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", "dev": true, "requires": { - "forwarded": "~0.1.2", + "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, @@ -11168,9 +11211,9 @@ "dev": true }, "qs": { - "version": "6.7.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", - "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==", + "version": "6.9.7", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.7.tgz", + "integrity": "sha512-IhMFgUmuNpyRfxA90umL7ByLlgRXu6tIfKPpF5TmcfRLlLCckfP/g3IQmju6jjpu+Hh8rA+2p6A27ZSPOOHdKw==", "dev": true }, "querystring": { @@ -11211,13 +11254,13 @@ "dev": true }, "raw-body": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz", - "integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==", + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.3.tgz", + "integrity": "sha512-UlTNLIcu0uzb4D2f4WltY6cVjLi+/jEN4lgEUj3E04tpMDpUlkBo/eSn6zou9hum2VMNpCCUone0O0WeJim07g==", "dev": true, "requires": { - "bytes": "3.1.0", - "http-errors": "1.7.2", + "bytes": "3.1.2", + "http-errors": "1.8.1", "iconv-lite": "0.4.24", "unpipe": "1.0.0" } @@ -11483,9 +11526,9 @@ "dev": true }, "send": { - "version": "0.17.1", - "resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz", - "integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==", + "version": "0.17.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.17.2.tgz", + "integrity": "sha512-UJYB6wFSJE3G00nEivR5rgWp8c2xXvJ3OPWPhmuteU0IKj8nKbG3DrjiOmLwpnHGYWAVwA69zmTm++YG0Hmwww==", "dev": true, "requires": { "debug": "2.6.9", @@ -11495,9 +11538,9 @@ "escape-html": "~1.0.3", "etag": "~1.8.1", "fresh": "0.5.2", - "http-errors": "~1.7.2", + "http-errors": "1.8.1", "mime": "1.6.0", - "ms": "2.1.1", + "ms": "2.1.3", "on-finished": "~2.3.0", "range-parser": "~1.2.1", "statuses": "~1.5.0" @@ -11515,15 +11558,15 @@ "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", "dev": true } } }, "ms": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", - "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "dev": true } } @@ -11535,15 +11578,15 @@ "dev": true }, "serve-static": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz", - "integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==", + "version": "1.14.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.2.tgz", + "integrity": "sha512-+TMNA9AFxUEGuC0z2mevogSnn9MXKb4fa7ngeRMJaaGv8vTwnIEkKi+QGvPt33HSnf8pRS+WGM0EbMtCJLKMBQ==", "dev": true, "requires": { "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "parseurl": "~1.3.3", - "send": "0.17.1" + "send": "0.17.2" } }, "set-blocking": { @@ -11582,9 +11625,9 @@ "dev": true }, "setprototypeof": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", - "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", "dev": true }, "sha.js": { @@ -12191,9 +12234,9 @@ } }, "toidentifier": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", - "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", "dev": true }, "tslib": { diff --git a/package.json b/package.json index 6d5084ebe4..311bda9a9a 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "ejs": "^3.1.7", "eslint": "^5.16.0", "eslint-loader": "^2.1.2", - "express": "^4.17.0", + "express": "^4.17.3", "fastestsmallesttextencoderdecoder": "^1.0.22", "git-revision-webpack-plugin": "^3.0.3", "google-closure-compiler": "^20210202.0.0", From 1c2d57136b6a7e2a18609761cf8be1dcd3d20fde Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 8 Jan 2023 17:48:25 +0000 Subject: [PATCH 106/137] build(deps): bump json5 from 1.0.1 to 1.0.2 Bumps [json5](https://github.com/json5/json5) from 1.0.1 to 1.0.2. - [Release notes](https://github.com/json5/json5/releases) - [Changelog](https://github.com/json5/json5/blob/main/CHANGELOG.md) - [Commits](https://github.com/json5/json5/compare/v1.0.1...v1.0.2) --- updated-dependencies: - dependency-name: json5 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- package-lock.json | 44 +++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/package-lock.json b/package-lock.json index e00d664613..b643d1a45a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3856,6 +3856,18 @@ "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=", "dev": true }, + "node_modules/json5": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz", + "integrity": "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==", + "dev": true, + "dependencies": { + "minimist": "^1.2.0" + }, + "bin": { + "json5": "lib/cli.js" + } + }, "node_modules/kind-of": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", @@ -3980,18 +3992,6 @@ "node": ">=4.0.0" } }, - "node_modules/loader-utils/node_modules/json5": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz", - "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", - "dev": true, - "dependencies": { - "minimist": "^1.2.0" - }, - "bin": { - "json5": "lib/cli.js" - } - }, "node_modules/locate-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", @@ -10205,6 +10205,15 @@ "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=", "dev": true }, + "json5": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz", + "integrity": "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==", + "dev": true, + "requires": { + "minimist": "^1.2.0" + } + }, "kind-of": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", @@ -10305,17 +10314,6 @@ "big.js": "^5.2.2", "emojis-list": "^3.0.0", "json5": "^1.0.1" - }, - "dependencies": { - "json5": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz", - "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", - "dev": true, - "requires": { - "minimist": "^1.2.0" - } - } } }, "locate-path": { From 1f77ea31f84979601673fa9e06f988f5cdef6887 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tarjei=20B=C3=A6rland?= Date: Mon, 16 Jan 2023 14:35:26 +0100 Subject: [PATCH 107/137] Remove dead link from readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 307e12f3eb..6d4f827f15 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ There is plenty of work still to do in making improvements to Skulpt. If you wo 1. Create a Github account if you don't already have one 2. Create a Fork of the Skulpt repository -- This will make a clone of the repository in your account. **DO NOT** clone this one. Once you've made the fork you will clone the forked version in your account to your local machine for development. -3. Read the HACKING.md file to get the "lay of the land". If you plan to work on creating a module then you may also find this [blog post](http://reputablejournal.com/adding-a-module-to-skulpt.html) helpful. +3. Read the HACKING.md file to get the "lay of the land". 3. Check the issues list for something to do. 4. Follow the instructions above to get skulpt building 5. Fix or add your own features. Commit and push to your forked version of the repository. When everything is tested and ready to be incorporated into the master version... From a5029b3b0f38611c6a3e35fa3dda4e96b4a81d54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tarjei=20B=C3=A6rland?= Date: Tue, 17 Jan 2023 08:22:20 +0100 Subject: [PATCH 108/137] Update link to adding a module in README.md As per [this comment](https://github.com/skulpt/skulpt/pull/1490#discussion_r1071368800). Also, there were two 3's in the numbered list. This commit updates the numbers to be consistent. --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 6d4f827f15..26f8b5307f 100644 --- a/README.md +++ b/README.md @@ -64,11 +64,11 @@ There is plenty of work still to do in making improvements to Skulpt. If you wo 1. Create a Github account if you don't already have one 2. Create a Fork of the Skulpt repository -- This will make a clone of the repository in your account. **DO NOT** clone this one. Once you've made the fork you will clone the forked version in your account to your local machine for development. -3. Read the HACKING.md file to get the "lay of the land". -3. Check the issues list for something to do. -4. Follow the instructions above to get skulpt building -5. Fix or add your own features. Commit and push to your forked version of the repository. When everything is tested and ready to be incorporated into the master version... -6. Make a Pull Request to get your feature(s) added to the main repository. +3. Read the HACKING.md file to get the "lay of the land". If you plan to work on creating a module then you may also find this [blog post](https://reputablejournal.com/2011/03/18/adding-a-module.html) helpful. +4. Check the issues list for something to do. +5. Follow the instructions above to get skulpt building +6. Fix or add your own features. Commit and push to your forked version of the repository. When everything is tested and ready to be incorporated into the master version... +7. Make a Pull Request to get your feature(s) added to the main repository. ## Community From 7184b3805082665e493e08c101c6a51b26d7cf55 Mon Sep 17 00:00:00 2001 From: stu Date: Tue, 17 Jan 2023 16:10:48 +0800 Subject: [PATCH 109/137] ffi: remove reference to window --- src/ffi.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ffi.js b/src/ffi.js index c1a99d0ef6..306f9a1426 100644 --- a/src/ffi.js +++ b/src/ffi.js @@ -685,7 +685,7 @@ const JsProxy = Sk.abstr.buildNativeClass("Proxy", { // neither do native js functions like requestAnimationFrame, JSON.parse // Proxy doesn't get a prototype but must be called with new - it's the only one I know // How you'd use Proxy in python I have no idea - return (this.is$type = jsFunc === window.Proxy); + return (this.is$type = jsFunc === Sk.global.Proxy); } const maybeConstructor = checkBodyIsMaybeConstructor(jsFunc); if (maybeConstructor === true) { From b27660fb27d586f1ded50f4c2db721abb2c618d7 Mon Sep 17 00:00:00 2001 From: Tim Huegerich Date: Tue, 31 Jan 2023 10:09:44 -0600 Subject: [PATCH 110/137] Correct typo in HACKING.md --- HACKING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HACKING.md b/HACKING.md index b0045bb24f..bea41ad974 100644 --- a/HACKING.md +++ b/HACKING.md @@ -807,7 +807,7 @@ skulpt.min.js and skulpt-stdlib.js A very minimal installation only uses skulpt.min.js, whereas if you want to use any modules they are in skulpt-stdlib.js. Looking around the distribution you will not immediately find skulpt.min.js because you need to build it. You get a -sculpt.js file by using the skulpty.py script that comes with the distribution. +sculpt.js file by using the skulpt.py script that comes with the distribution. running `./skulpt.py --help` will give you the full list of commands, but the two that you probably most care about are `npm run build` and `npm run docbi` The dist command builds both skulpt.min.js and skulpt-stdlib.js docbi builds From 5559418f56528b340851699b53a6db8bce5ed8de Mon Sep 17 00:00:00 2001 From: mrcork Date: Mon, 5 Oct 2020 14:24:55 +0800 Subject: [PATCH 111/137] re module updated --- src/lib/re.js | 1485 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 1089 insertions(+), 396 deletions(-) diff --git a/src/lib/re.js b/src/lib/re.js index ae2b1f0558..5fc524466c 100644 --- a/src/lib/re.js +++ b/src/lib/re.js @@ -1,435 +1,1128 @@ -var $builtinmodule = function (name) { - var mod = {__name__: new Sk.builtin.str("re")}; - - var validGroups, convert, getFlags, _split, _findall, matchobj, _search, _match, regexobj; - - // Constants (mostly unsupported) - mod.I = 2; - mod.IGNORECASE = 2; - // mod.L = 4; - // mod.LOCALE = 4; - mod.M = 8; - mod.MULTILINE = 8; - // mod.S = 16; - // mod.DOTALL = 16; - // mod.U = 32; - // mod.UNICODE = 32; - // mod.X = 64; - // mod.VERBOSE = 64; - - validGroups = ["(?:", "(?=", "(?!"]; - - convert = function (pattern) { - var newpattern; - var match; - var i; +function $builtinmodule(name) { + const re = { + __name__: new Sk.builtin.str("re"), + __all__: new Sk.builtin.list( + [ + "match", + "fullmatch", + "search", + "sub", + "subn", + "split", + "findall", + "finditer", + "compile", + "purge", + "template", + "escape", + "error", + "Pattern", + "Match", + "A", + "I", + "L", + "M", + "S", + "X", + "U", + "ASCII", + "IGNORECASE", + "LOCALE", + "MULTILINE", + "DOTALL", + "VERBOSE", + "UNICODE", + ].map((x) => new Sk.builtin.str(x)) + ), + }; - // Look for disallowed constructs - match = pattern.match(/\(\?./g); - if (match) { - for (i = 0; i < match.length; i++) { - if (validGroups.indexOf(match[i]) == -1) { - throw new Sk.builtin.ValueError("Disallowed group in pattern: '" - + match[i] + "'"); + // cached flags + const _value2member = {}; + + const RegexFlagMeta = Sk.abstr.buildNativeClass("RegexFlagMeta", { + constructor: function RegexFlagMeta() {}, + base: Sk.builtin.type, + slots: { + tp$iter() { + const members = Object.values(_members)[Symbol.iterator](); + return new Sk.misceval.iterator(() => members.next().value); + }, + sq$contains(flag) { + if (!(flag instanceof this)) { + throw new Sk.builtin.TypeError( + "unsupported operand type(s) for 'in': '" + Sk.abstr.typeName(flag) + "' and '" + Sk.abstr.typeName(this) + "'" + ); } - } - } + return Object.values(_members).includes(flag); + }, + }, + }); - newpattern = pattern.replace("/\\/g", "\\\\"); - newpattern = pattern.replace(/([^\\]){,(?![^\[]*\])/g, "$1{0,"); + re.RegexFlag = Sk.abstr.buildNativeClass("RegexFlag", { + meta: RegexFlagMeta, + base: Sk.builtin.int_, + constructor: function RegexFlag(value) { + const member = _value2member[value]; + if (member) { + return member; + } + this.v = value; + _value2member[value] = this; + }, + + slots: { + tp$new(args, kwargs) { + Sk.abstr.checkOneArg("RegexFlag", args, kwargs); + const value = args[0].valueOf(); + if (!Sk.builtin.checkInt(value)) { + throw new Sk.builtin.ValueError(Sk.abstr.objectRepr(value) + " is not a valid RegexFlag"); + } + return new re.RegexFlag(value); + }, + $r() { + let value = this.valueOf(); + const neg = value < 0; + value = neg ? ~value : value; + const members = []; + Object.entries(_members).forEach(([name, m]) => { + // we're not supporting bigints here seems sensible not to + const m_value = m.valueOf(); + if (value & m_value) { + value &= ~m_value; + members.push("re." + name); + } + }); + if (value) { + members.push(Sk.builtin.hex(value).toString()); + } + let res = members.join("|"); - return newpattern; - }; + if (neg) { + res = members.length > 1 ? "~(" + res + ")" : "~" + res; + } + return new Sk.builtin.str(res); + }, + sq$contains(flag) { + if (!(flag instanceof re.RegexFlag)) { + throw new Sk.builtin.TypeError("'in' requires a RegexFlag not " + Sk.abstr.typeName(flag)); + } + return this.nb$and(flag) === flag; + }, + nb$and: flagBitSlot((v, w) => v & w, JSBI.bitwiseAnd), + nb$or: flagBitSlot((v, w) => v | w, JSBI.bitwiseOr), + nb$xor: flagBitSlot((v, w) => v ^ w, JSBI.bitwiseXor), + nb$invert: function () { + const v = this.v; + if (typeof v === "number") { + return new re.RegexFlag(~v); + } + return new re.RegexFlag(JSBI.bitwiseNot(v)); + }, + }, + proto: { + valueOf() { + return this.v; + }, + }, + flags: { + sk$acceptable_as_base_class: false, + }, + }); - getFlags = function (flags) { - var jsflags = "g"; - if ((flags & mod.IGNORECASE) == mod.IGNORECASE) { - jsflags += "i"; - } - if ((flags & mod.MULTILINE) == mod.MULTILINE) { - jsflags += "m"; - } - return jsflags; + re.TEMPLATE = re.T = new re.RegexFlag(1); + re.IGNORECASE = re.I = new re.RegexFlag(2); + re.LOCALE = re.L = new re.RegexFlag(4); + re.MULTILINE = re.M = new re.RegexFlag(8); + re.DOTALL = re.S = new re.RegexFlag(16); + re.UNICODE = re.U = new re.RegexFlag(32); + re.VERBOSE = re.X = new re.RegexFlag(64); + re.DEBUG = new re.RegexFlag(128); + re.ASCII = re.A = new re.RegexFlag(256); + + const _members = { + ASCII: re.A, + IGNORECASE: re.I, + LOCALE: re.L, + UNICODE: re.U, + MULTILINE: re.M, + DOTALL: re.S, + VERBOSE: re.X, + TEMPLATE: re.T, + DEBUG: re.DEBUG, }; - _split = function (pattern, string, maxsplit, flags) { - var pat, str, captured, jsflags, regex; - var result, match, index, splits; - - Sk.builtin.pyCheckArgsLen("split", arguments.length, 2, 4); - if (!Sk.builtin.checkString(pattern)) { - throw new Sk.builtin.TypeError("pattern must be a string"); - } - if (!Sk.builtin.checkString(string)) { - throw new Sk.builtin.TypeError("string must be a string"); - } - if (maxsplit === undefined) { - maxsplit = 0; - } - if (!Sk.builtin.checkNumber(maxsplit)) { - throw new Sk.builtin.TypeError("maxsplit must be a number"); - } - if (flags === undefined) { - flags = 0; - } - if (!Sk.builtin.checkNumber(flags)) { - throw new Sk.builtin.TypeError("flags must be a number"); - } - - maxsplit = Sk.builtin.asnum$(maxsplit); - pat = Sk.ffi.remapToJs(pattern); - str = Sk.ffi.remapToJs(string); - - // Convert pat from Python to Javascript regex syntax - pat = convert(pat); - //print("Pat: " + pat); - //print("Str: " + str); - - captured = !(pat.match(/^\(.*\)$/) === null); - //print("Captured: ", captured); - - jsflags = getFlags(flags); - //print("Flags: ", jsflags); - - regex = new RegExp(pat, jsflags); - - result = []; - match; - index = 0; - splits = 0; - while ((match = regex.exec(str)) != null) { - //print("Matched '" + match[0] + "' at position " + match.index + - // "; next search at " + regex.lastIndex); - if (match.index === regex.lastIndex) { - // empty match - break; - } - result.push(new Sk.builtin.str(str.substring(index, match.index))); - if (captured) { - // Add matching pattern, too - result.push(new Sk.builtin.str(match[0])); - } - index = regex.lastIndex; - splits += 1; - if (maxsplit && (splits >= maxsplit)) { - break; + function flagBitSlot(number_func, bigint_func) { + return function (other) { + if (other instanceof re.RegexFlag || other instanceof Sk.builtin.int_) { + let v = this.v; + let w = other.v; + if (typeof v === "number" && typeof w === "number") { + let tmp = number_func(v, w); + if (tmp < 0) { + tmp = tmp + 4294967296; // convert back to unsigned + } + return new re.RegexFlag(tmp); + } + v = JSBI.BigUp(v); + w = JSBI.BigUp(w); + return new re.RegexFlag(JSBI.numberIfSafe(bigint_func(v, w))); } - } - result.push(new Sk.builtin.str(str.substring(index))); + return Sk.builtin.NotImplemented.NotImplemented$; + }; + } - return new Sk.builtin.list(result); + const jsFlags = { + i: re.I, + m: re.M, + s: re.S, + u: re.U, + }; + const jsInlineFlags = { + i: re.I, + a: re.A, + s: re.S, + L: re.L, + m: re.M, + u: re.U, + x: re.X, }; - _split.co_varnames = ["pattern", "string", "maxsplit", "flags"]; - _split.$defaults = [ new Sk.builtin.int_(0), new Sk.builtin.int_(0) ]; - - mod.split = new Sk.builtin.func(_split); - - _findall = function (pattern, string, flags) { - var pat, str, jsflags, regex, result, match; - - Sk.builtin.pyCheckArgsLen("findall", arguments.length, 2, 3); - if (!Sk.builtin.checkString(pattern)) { - throw new Sk.builtin.TypeError("pattern must be a string"); - } - if (!Sk.builtin.checkString(string)) { - throw new Sk.builtin.TypeError("string must be a string"); - } - if (flags === undefined) { - flags = 0; + (function checkJsFlags() { + try { + new RegExp("", "u"); + } catch { + delete jsFlags["u"]; } - if (!Sk.builtin.checkNumber(flags)) { - throw new Sk.builtin.TypeError("flags must be a number"); + try { + new RegExp("", "s"); + } catch { + delete jsFlags["s"]; } + })(); - pat = Sk.ffi.remapToJs(pattern); - str = Sk.ffi.remapToJs(string); - - // Convert pat from Python to Javascript regex syntax - pat = convert(pat); - //print("Pat: " + pat); - //print("Str: " + str); - - jsflags = getFlags(flags); - //print("Flags: ", jsflags); - - regex = new RegExp(pat, jsflags); - - if (pat.match(/\$/)) { - var newline_at_end = new RegExp(/\n$/); - if (str.match(newline_at_end)) { - str = str.slice(0, -1); + const flagFails = Object.entries({ + "cannot use LOCALE flag with a str pattern": re.L, + "ASCII and UNICODE flags are incompatible": new re.RegexFlag(re.A.valueOf() | re.U.valueOf()), + }); + const inline_regex = /\(\?([i|s|a|m|u|x]+)\)/g; + + function adjustFlags(pyPattern, pyFlag) { + let jsPattern = pyPattern.toString(); + let jsFlag = "g"; + // currently not worrying about bytes; + // need to check compatibility of auL - also L not valid for str patterns + let inlineFlags = 0; + jsPattern = jsPattern.replace(inline_regex, (match, inline) => { + for (let i of inline) { + const inlineFlag = jsInlineFlags[i]; + inlineFlags = inlineFlags | inlineFlag.valueOf(); } - } + return ""; + }); - result = []; - match; - while ((match = regex.exec(str)) != null) { - //print("Matched '" + match[0] + "' at position " + match.index + - // "; next search at " + regex.lastIndex); - // print("match: " + JSON.stringify(match)); - if (match.length < 2) { - result.push(new Sk.builtin.str(match[0])); - } else if (match.length == 2) { - result.push(new Sk.builtin.str(match[1])); - } else { - var groups = []; - for (var i = 1; i < match.length; i++) { - groups.push(new Sk.builtin.str(match[i])); - } - result.push(new Sk.builtin.tuple(groups)); - } - if (match.index === regex.lastIndex) { - regex.lastIndex += 1; + // check if inlineFlags (it throws a different error) + flagFails.forEach(([msg, flag]) => { + if ((flag.valueOf() & inlineFlags) === flag.valueOf()) { + throw new re.error("bad bad inline flags: " + msg); } - } - - return new Sk.builtin.list(result); - }; - - _findall.co_varnames = ["pattern", "string", "flags"]; - _findall.$defaults = [ new Sk.builtin.int_(0) ]; - - mod.findall = new Sk.builtin.func(_findall); - - - matchobj = function ($gbl, $loc) { - $loc.__init__ = new Sk.builtin.func(function (self, thematch, pattern, string) { - self.thematch = thematch; - self.re = pattern; - self.string = string; - return Sk.builtin.none.none$; }); - $loc.groups = new Sk.builtin.func(function (self) { - var _groups = self.thematch.v.slice(1); + pyFlag = Sk.abstr.numberBinOp(new re.RegexFlag(inlineFlags), pyFlag, "BitOr"); - return new Sk.builtin.tuple(_groups); - }); - - $loc.group = new Sk.builtin.func(function (self, grpnum) { - if (grpnum === undefined) { - grpnum = 0; - } else { - grpnum = Sk.builtin.asnum$(grpnum); + // check compatibility of flags + flagFails.forEach(([msg, flag]) => { + if (Sk.abstr.numberBinOp(flag, pyFlag, "BitAnd") === flag) { + throw new Sk.builtin.ValueError(msg); } - if (grpnum >= self.thematch.v.length) { - throw new Sk.builtin.IndexError("Index out of range: " + grpnum); - } - return self.thematch.v[grpnum]; }); - }; - - mod.MatchObject = Sk.misceval.buildClass(mod, matchobj, "MatchObject", []); - - // Internal function to return a Python list of strings - // From a JS regular expression string - mod._findre = function (res, string) { - res = res.replace(/([^\\]){,(?![^\[]*\])/g, "$1{0,"); - - var matches, sitem, retval; - var re = eval(res); - var patt = new RegExp("\n$"); - var str = Sk.ffi.remapToJs(string); - - if (str.match(patt)) { - matches = str.slice(0, -1).match(re); - } else { - matches = str.match(re); - } - retval = new Sk.builtin.list(); - if (matches == null) { - return retval; - } - for (var i = 0; i < matches.length; ++i) { - sitem = new Sk.builtin.str(matches[i]); - retval.v.push(sitem); - } - return retval; - }; - - - // Internal search, shared between search function and RegexObject.search method - _search = function (pattern, string, flags) { - var mob, res; - - Sk.builtin.pyCheckArgsLen("search", arguments.length, 2, 3); - if (!Sk.builtin.checkString(pattern)) { - throw new Sk.builtin.TypeError("pattern must be a string"); + // use unicode? + if (Sk.abstr.numberBinOp(re.A, pyFlag, "BitAnd") !== re.A) { + pyFlag = Sk.abstr.numberBinOp(re.U, pyFlag, "BitOr"); } - if (!Sk.builtin.checkString(string)) { - throw new Sk.builtin.TypeError("string must be a string"); - } - if (flags === undefined) { - flags = 0; - } - if (!Sk.builtin.checkNumber(flags)) { - throw new Sk.builtin.TypeError("flags must be a number"); - } - res = "/" + pattern.v.replace(/\//g, "\\/") + "/"; - const lst = mod._findre(res, string); - if (lst.v.length < 1) { - return Sk.builtin.none.none$; - } - mob = Sk.misceval.callsimArray(mod.MatchObject, [lst, pattern, string]); - return mob; - }; - _search.co_varnames = ["pattern", "string", "flags"]; - _search.$defaults = [ new Sk.builtin.int_(0) ]; + Object.entries(jsFlags).forEach(([flag, reFlag]) => { + if (Sk.abstr.numberBinOp(reFlag, pyFlag, "BitAnd") === reFlag) { + jsFlag += flag; + } + }); + pyFlag = new re.RegexFlag(pyFlag.valueOf()); // just incase we're an integer - mod.search = new Sk.builtin.func(_search); + return [jsPattern, jsFlag, pyFlag]; + } - _match = function (pattern, string, flags) { - var mob, res; - Sk.builtin.pyCheckArgsLen("match", arguments.length, 2, 3); - if (!Sk.builtin.checkString(pattern)) { - throw new Sk.builtin.TypeError("pattern must be a string"); - } - if (!Sk.builtin.checkString(string)) { - throw new Sk.builtin.TypeError("string must be a string"); - } - if (flags === undefined) { - flags = 0; + let neg_lookbehind_A = "(? + const py_to_js_regex = /([^\\])({,|\\A|\\Z|\$|\(\?P=([^\d\W]\w*)\)|\(\?P<([^\d\W]\w*)>)(?![^\[]*\])/g; + // unicode mode in js regex treats \\\t incorrectly and should be converted to \\t + // similarly \" and \' \! \& throw errors + const py_to_js_unicode_escape = /\\[\t\r\n \v\f#&~"'!]|\\-(?![^\[]*\])/g; + const quantifier_error = /Incomplete quantifier|Lone quantifier/g; + + const _compiled_patterns = Object.create(null); + + function compile_pattern(pyPattern, pyFlag) { + let jsPattern, jsFlags; + [jsPattern, jsFlags, pyFlag] = adjustFlags(pyPattern, pyFlag); + const _cached = _compiled_patterns[pyPattern.toString()]; + if (_cached && _cached.$flags === pyFlag) { + return _cached; } - const pat = Sk.ffi.remapToJs(pattern); - res = "/^" + pat.replace(/\//g, "\\/") + "/"; - const lst = mod._findre(res, string); - if (Sk.ffi.remapToJs(lst).length < 1) { - return Sk.builtin.none.none$; - } - mob = Sk.misceval.callsimArray(mod.MatchObject, [lst, pattern, string]); - return mob; - }; - - _match.co_varnames = ["pattern", "string", "flags"]; - _match.$defaults = [ new Sk.builtin.int_(0) ]; - - mod.match = new Sk.builtin.func(_match); - - regexobj = function ($gbl, $loc) { - var _slice, _re_search, _re_match, _re_split, _re_findall, _repr; - $loc.__init__ = new Sk.builtin.func(function (self, pattern, flags) { - self.re = pattern; - if (flags === undefined) { - self.flags = 0; - } else { - self.flags = flags; + const named_groups = {}; + jsPattern = "_" + jsPattern; // prepend so that we can safely not use negative lookbehinds + jsPattern = jsPattern.replace(py_to_js_regex, (m, p0, p1, p2, p3, offset) => { + switch (p1) { + case "\\A": + return p0 + neg_lookbehind_A + "^"; + case "\\Z": + return p0 + "$(?!\\n)"; + case "{,": + return p0 + "{0,"; + case "$": + return p0 + "(?:(?=\\n$)|$)"; + default: + if (p1.endsWith(">")) { + named_groups[p3] = true; + return p0 + "(?<" + p3 + ">"; + } + if (!named_groups[p2]) { + throw new re.error("unknown group name " + p2 + " at position " + offset + 1, pyPattern, new Sk.builtin.int_(offset + 1)); + } + return p0 + "\\k<" + p2 + ">"; } - return Sk.builtin.none.none$; - }); - - _repr = new Sk.builtin.func( function (self) { - var ret = "re.compile('" + Sk.ffi.remapToJs(self.re) + "')"; - return Sk.ffi.remapToPy(ret.substring(0,212)); }); - - $loc.__str__ = _repr; - - $loc.__repr__ = _repr; - - // Given a string, start, and end position, return sliced string - _slice = function(string, pos, endpos) { - // Per docs, ^ should match index after newlines. - // this matches the first - var str = Sk.ffi.remapToJs(string); - var start = pos == undefined ? 0 : Sk.ffi.remapToJs(pos); - var end = endpos == undefined ? str.length : Sk.ffi.remapToJs(endpos); - - if (start == "^") { - start = str.indexOf("\n") + 1; - } - if (end === null) { - end = str.length; - } - return Sk.ffi.remapToPy(str.substring(start, end)); - - }; - - _re_search = function (self, string, pos, endpos) { - Sk.builtin.pyCheckArgsLen("search", arguments.length, 2, 4); - - var str = _slice(string, pos, endpos); - - return _search(self.re, str, self.flags); - }; - - _re_search.co_varnames = ["self", "string", "pos", "endpos"]; - _re_search.$defaults = [ new Sk.builtin.int_(0), Sk.builtin.none.none$ ]; - - $loc.search = new Sk.builtin.func(_re_search); - - _re_match = function (self, string, pos, endpos) { - Sk.builtin.pyCheckArgsLen("match", arguments.length, 2, 4); - - var str = _slice(string, pos, endpos); - // var str = string; - - return _match(self.re, str, self.flags); - }; - - _re_match.co_varnames = ["self", "string", "pos", "endpos"]; - _re_match.$defaults = [ new Sk.builtin.int_(0), Sk.builtin.none.none$ ]; - - $loc.match = new Sk.builtin.func(_re_match); - - _re_split = function (self, string, maxsplit) { - Sk.builtin.pyCheckArgsLen("split", arguments.length, 2, 3); - - if (maxsplit === undefined) { - maxsplit = 0; - } - if (!Sk.builtin.checkInt(maxsplit)) { - throw new Sk.builtin.TypeError("maxsplit must be an integer"); + jsPattern = jsPattern.slice(1); + let regex; + let msg; + let unicodeEscapedPattern = jsPattern; + debugger; + if (jsFlags.includes("u")) { + // then we we need to adjust the escapes for \\\t to be \\t etc because javascript reads escapes differently in unicode mode! + // '\\-' is different - inside a square bracket it gets compiled but outside it doesn't! + unicodeEscapedPattern = jsPattern.replace(py_to_js_unicode_escape, (m) => { + switch (m) { + case "\\ ": + return " "; + case "\\\t": + return "\\t"; + case "\\\n": + debugger; + return "\\n"; + case "\\\v": + return "\\v"; + case "\\\f": + return "\\f"; + case "\\r": + return "\\r"; + default: + return m.slice(1); + } + }); + } + try { + regex = new RegExp(unicodeEscapedPattern, jsFlags); + } catch (e) { + if (quantifier_error.test(e.message)) { + try { + // try without the unicode flag + regex = new RegExp(jsPattern, jsFlags.replace("u","")); + } catch (e) { + msg = e.message.substring(e.message.lastIndexOf(":") + 2) + " in pattern: " + pyPattern.toString(); + throw new re.error(msg, pyPattern); + } + } else { + msg = e.message.substring(e.message.lastIndexOf(":") + 2) + " in pattern: " + pyPattern.toString(); + throw new re.error(msg, pyPattern); } - - return _split(self.re, string, maxsplit, self.flags); - }; - - _re_split.co_varnames = ["self", "string", "maxsplit"]; - _re_split.$defaults = [ new Sk.builtin.int_(0) ]; - - $loc.split = new Sk.builtin.func(_re_split); - - _re_findall = function (self, string, pos, endpos) { - Sk.builtin.pyCheckArgsLen("findall", arguments.length, 2, 4); - - var str = _slice(string, pos, endpos); - - return _findall(self.re, str, self.flags); - }; - - _re_findall.co_varnames = ["self", "string", "pos", "endpos"]; - _re_findall.$defaults = [ new Sk.builtin.int_(0), Sk.builtin.none.none$ ]; - - $loc.findall = new Sk.builtin.func(_re_findall); - - }; - - mod.RegexObject = Sk.misceval.buildClass(mod, regexobj, "RegexObject", []); - mod.compile = new Sk.builtin.func(function (pattern, flags) { - var rob; - Sk.builtin.pyCheckArgsLen("compile", arguments.length, 1, 2); - if (!Sk.builtin.checkString(pattern)) { - throw new Sk.builtin.TypeError("pattern must be a string"); } - if (flags === undefined) { - flags = 0; + const ret = new re.Pattern(regex, pyPattern, pyFlag); + _compiled_patterns[pyPattern.toString()] = ret; + return ret; + } + + function _compile(pattern, flag) { + if (pattern instanceof re.Pattern) { + if (flag !== zero || flag.valueOf()) { + throw new Sk.builtin.ValueError("cannot process flags argument with compiled pattern"); + } + return pattern; } - if (!Sk.builtin.checkNumber(flags)) { - throw new Sk.builtin.TypeError("flags must be a number"); + if (!Sk.builtin.checkString(pattern)) { + throw new Sk.builtin.TypeError("first argument must be string or compiled pattern"); } - rob = Sk.misceval.callsimArray(mod.RegexObject, [pattern, flags]); - return rob; + return compile_pattern(pattern, flag); // compile the pattern to javascript Regex + } + + re.error = Sk.abstr.buildNativeClass("re.error", { + base: Sk.builtin.Exception, + constructor: function error(msg, pattern, pos) { + this.$pattern = pattern; + this.$msg = msg; + this.$pos = pos || Sk.builtin.none.none$; + Sk.builtin.Exception.call(this, msg); + }, + slots: { + tp$doc: + "Exception raised for invalid regular expressions.\n\n Attributes:\n\n msg: The unformatted error message\n pattern: The regular expression pattern\n", + tp$init(args, kwargs) { + const [msg, pattern, pos] = Sk.abstr.copyKeywordToNamedArgs("re.error", ["msg", "pattern", "pos"], args, kwargs, [ + Sk.builtin.none.none$, + Sk.builtin.none.none$, + ]); + this.$pattern = pattern; + this.$pos = pos; + this.$msg = msg; + }, + }, + getsets: { + msg: { + $get() { + return this.$msg; + }, + }, + pattern: { + $get() { + return this.$pattern; + }, + }, + pos: { + $get() { + return this.$pos; + }, + }, + }, }); - // No need to purge since we don't cache - mod.purge = new Sk.builtin.func(function () {}); + const zero = new Sk.builtin.int_(0); + const maxsize = Number.MAX_SAFE_INTEGER; + + re.Pattern = Sk.abstr.buildNativeClass("re.Pattern", { + constructor: function (regex, str, flags) { + this.v = regex; + this.str = str; + this.$flags = flags; + this.$groups = null; + this.$groupindex = null; + }, + slots: { + $r() { + const patrepr = Sk.misceval.objectRepr(this.str).slice(0, 200); + const flagrepr = Sk.misceval.objectRepr(this.$flags.nb$and(re.U.nb$invert())); // re.U is not included in the repr here + return new Sk.builtin.str("re.compile(" + patrepr + (flagrepr ? ", " + flagrepr : "") + ")"); + }, + tp$richcompare(other, op) { + if ((op !== "Eq" && op !== "NotEq") || !(other instanceof re.Pattern)) { + return Sk.builtin.NotImplemented.NotImplemented$; + } + const res = this.str === other.str && this.$flags === other.$flags; + return op === "Eq" ? res : !res; + }, + tp$hash() {}, + tp$doc: "Compiled regular expression object.", + }, + methods: { + match: { + $meth: function match(string, pos, endpos) { + return this.$match(string, pos, endpos); + }, + $flags: { NamedArgs: ["string", "pos", "endpos"], Defaults: [zero, maxsize] }, + $textsig: "($self, /, string, pos=0, endpos=sys.maxsize)", + $doc: "Matches zero or more characters at the beginning of the string.", + }, + fullmatch: { + $meth: function fullmatch(string, pos, endpos) { + return this.full$match(string, pos, endpos); + }, + $flags: { NamedArgs: ["string", "pos", "endpos"], Defaults: [zero, maxsize] }, + $textsig: "($self, /, string, pos=0, endpos=sys.maxsize)", + $doc: "Matches against all of the string.", + }, + search: { + $meth: function search(string, pos, endpos) { + return this.$search(string, pos, endpos); + }, + $flags: { NamedArgs: ["string", "pos", "endpos"], Defaults: [zero, maxsize] }, + $textsig: "($self, /, string, pos=0, endpos=sys.maxsize)", + $doc: + "Scan through string looking for a match, and return a corresponding match object instance.\n\nReturn None if no position in the string matches.", + }, + sub: { + $meth: function sub(repl, string, count) { + return this.$sub(repl, string, count); + }, + $flags: { NamedArgs: ["repl", "string", "count"], Defaults: [zero] }, + $textsig: "($self, /, repl, string, count=0)", + $doc: + "Return the string obtained by replacing the leftmost non-overlapping occurrences of pattern in string by the replacement repl.", + }, + subn: { + $meth: function (repl, string, count) { + return this.$subn(repl, string, count); + }, + $flags: { NamedArgs: ["repl", "string", "count"], Defaults: [zero] }, + $textsig: "($self, /, repl, string, count=0)", + $doc: + "Return the tuple (new_string, number_of_subs_made) found by replacing the leftmost non-overlapping occurrences of pattern with the replacement repl.", + }, + findall: { + $meth: function findall(string, pos, endpos) { + return this.find$all(string, pos, endpos); + }, + $flags: { NamedArgs: ["string", "pos", "endpos"], Defaults: [zero, maxsize] }, + $textsig: "($self, /, string, pos=0, endpos=sys.maxsize)", + $doc: "Return a list of all non-overlapping matches of pattern in string.", + }, + split: { + $meth: function split(string, maxsplit) { + return this.$split(string, maxsplit); + }, + $flags: { NamedArgs: ["string", "maxsplit"], Defaults: [zero] }, + $textsig: "($self, /, string, maxsplit=0)", + $doc: "Split string by the occurrences of pattern.", + }, + finditer: { + $meth: function finditer(string, pos, endpos) { + return this.find$iter(string, pos, endpos); + }, + $flags: { NamedArgs: ["string", "pos", "endpos"], Defaults: [zero, maxsize] }, + $textsig: "($self, /, string, pos=0, endpos=sys.maxsize)", + $doc: + "Return an iterator over all non-overlapping matches for the RE pattern in string.\n\nFor each match, the iterator returns a match object.", + }, + scanner: { + $meth: function scanner(string, pos, endpos) { + return this.$scanner(string, pos, endpos); + }, + $flags: { NamedArgs: ["string", "pos", "endpos"], Defaults: [zero, maxsize] }, + $textsig: "($self, /, string, pos=0, endpos=sys.maxsize)", + $doc: null, + }, + __copy__: { + $meth: function copy() { + return this; + }, + $flags: { NoArgs: true }, + $textsig: "($self, /)", + $doc: null, + }, + __deepcopy__: { + $meth: function () { + return this; + }, + $flags: { OneArg: true }, + $textsig: "($self, memo, /)", + $doc: null, + }, + }, + getsets: { + pattern: { + $get() { + return this.str; + }, + $doc: "The pattern string from which the RE object was compiled.", + }, + flags: { + $get() { + return this.$flags; + }, + $doc: "The regex matching flags.", + }, + groups: { + $get() { + if (this.$groups === null) { + // we know we have a compiled expression so we just need to check matching brackets + // bracket characters that are not inside [] not followed by ? but could be followed by ?P< + const num_matches = (this.str.v.match(this.group$regex) || []).length; + this.$groups = new Sk.builtin.int_(num_matches); + } + return this.$groups; + }, + $doc: "The number of capturing groups in the pattern.", + }, + groupindex: { + $get() { + if (this.$groupindex === null) { + const matches = this.str.v.matchAll(this.group$regex); + const arr = []; + let i = 1; + for (match of matches) { + if (match[1]) { + arr.push(new Sk.builtin.str(match[1])); + arr.push(new Sk.builtin.int_(i)); + } + i++; + } + this.$groupindex = new Sk.builtin.mappingproxy(new Sk.builtin.dict(arr)); + } + return this.$groupindex; + }, + $doc: "A dictionary mapping group names to group numbers.", + }, + }, + proto: { + // Any opening bracket not inside [] Not followed by ? but might could be followed by ?P + // if it's a group like (?P) then we need to capture the foo + group$regex: /\((?!\?(?!P<).*)(?:\?P<([^\d\W]\w*)>)?(?![^\[]*\])/g, + get$count(count) { + count = Sk.misceval.asIndexSized(count, Sk.builtin.OverflowError); + return count ? count : Number.POSITIVE_INFINITY; + }, + get$jsstr(string, pos, endpos) { + if (!Sk.builtin.checkString(string)) { + throw new Sk.builtin.TypeError("expected string or bytes-like object"); + } + if ((pos === zero && endpos === maxsize) || (pos === undefined && endpos === undefined)) { + return { jsstr: string.toString(), pos: zero.valueOf(), endpos: string.sq$length() }; + } + const { start, end } = Sk.builtin.slice.startEnd$wrt(string, pos, endpos); + return { jsstr: string.toString().slice(start, end), pos: start, endpos: end }; + }, + find$all(string, pos, endpos) { + let { jsstr } = this.get$jsstr(string, pos, endpos); + const regex = this.v; + const matches = jsstr.matchAll(regex); + const ret = []; + for (let match of matches) { + // do we have groups? + ret.push( + match.length === 1 + ? new Sk.builtin.str(match[0]) + : match.length === 2 + ? new Sk.builtin.str(match[1]) + : new Sk.builtin.tuple(match.slice(1).map((x) => new Sk.builtin.str(x))) + ); + } + return new Sk.builtin.list(ret); + }, + $split(string, maxsplit) { + maxsplit = Sk.misceval.asIndexSized(maxsplit); + maxsplit = maxsplit ? maxsplit : Number.POSITIVE_INFINITY; + let { jsstr } = this.get$jsstr(string); + const regex = this.v; + const split = []; + let match; + let num_splits = 0; + let idx = 0; + while ((match = regex.exec(jsstr)) !== null && num_splits < maxsplit) { + split.push(new Sk.builtin.str(jsstr.substring(idx, match.index))); + if (match.length > 1) { + split.push(...match.slice(1).map((x) => (x === undefined ? Sk.builtin.none.none$ : new Sk.builtin.str(x)))); + } + num_splits++; + idx = regex.lastIndex; + if (match.index === regex.lastIndex) { + if (jsstr) { + jsstr = jsstr.slice(match.index); + // need to reset the regex.lastIndex; + idx = 0; + regex.lastIndex = 1; + } else { + break; // check this; + } + } + } + regex.lastIndex = 0; + split.push(new Sk.builtin.str(jsstr.slice(idx))); + return new Sk.builtin.list(split); + }, + match$from_repl(args, string, pos, endpos) { + let match_like; + const named_groups = args[args.length - 1]; + if (typeof named_groups === "object") { + match_like = args.slice(0, args.length - 3); + Object.assign(match_like, { groups: named_groups }); + match_like.index = args[args.length - 3]; + } else { + match_like = args.slice(0, args.length - 2); + match_like.groups = undefined; + match_like.index = args[args.length - 2]; + } + return new re.Match(match_like, this.str, string, pos, endpos); + }, + do$sub(repl, string, count) { + const { jsstr, pos, endpos } = this.get$jsstr(string); + let matchRepl; + if (Sk.builtin.checkCallable(repl)) { + matchRepl = (matchObj) => { + const rep = Sk.misceval.callsimArray(repl, [matchObj]); + if (!Sk.builtin.checkString(rep)) { + throw new Sk.builtin.TypeError("expected str instance, " + Sk.abstr.typeName(rep) + " found"); + } + return rep.toString(); + }; + } else { + repl = this.get$jsstr(repl).jsstr; + matchRepl = (matchObj) => matchObj.template$repl(repl); + } + count = this.get$count(count); + let num_repl = 0; + const ret = jsstr.replace(this.v, (...args) => { + if (num_repl >= count) { + return args[0]; + } + num_repl++; + const matchObj = this.match$from_repl(args, string, pos, endpos); + return matchRepl(matchObj); + }); + return [new Sk.builtin.str(ret), new Sk.builtin.int_(num_repl)]; + }, + $sub(repl, string, count) { + const [ret] = this.do$sub(repl, string, count); + return ret; + }, + $subn(repl, string, count) { + return new Sk.builtin.tuple(this.do$sub(repl, string, count)); + }, + do$match(regex, string, pos, endpos) { + let jsstr; + ({ jsstr, pos, endpos } = this.get$jsstr(string, pos, endpos)); + const match = jsstr.match(regex); + if (match === null) { + return Sk.builtin.none.none$; + } + return new re.Match(match, this, string, pos, endpos); + }, + $search(string, pos, endpos) { + var regex = new RegExp(this.v.source, this.v.flags.replace("g", "")); // replace all flags except 'g'; + return this.do$match(regex, string, pos, endpos); + }, + $match(string, pos, endpos) { + let source = this.v.source; + let flags = this.v.flags.replace("g", "").replace("m", ""); + source = "^" + source; + var regex = new RegExp(source, flags); + return this.do$match(regex, string, pos, endpos); + }, + full$match(string, pos, endpos) { + let source = this.v.source; + let flags = this.v.flags.replace("g", "").replace("m", ""); + source = "^(?:" + source + ")$"; + var regex = new RegExp(source, flags); + return this.do$match(regex, string, pos, endpos); + }, + find$iter(string, pos, endpos) { + let jsstr; + ({ jsstr, pos, endpos } = this.get$jsstr(string, pos, endpos)); + const matchIter = jsstr.matchAll(this.v); + return new Sk.misceval.iterator(() => { + const match = matchIter.next().value; + if (match === undefined) { + return undefined; + } + return new re.Match(match, this, string, pos, endpos); + }); + // could adjust this to use exec. + }, + }, + flags: { + sk$acceptable_as_base_class: false, + }, + }); + + re.Match = Sk.abstr.buildNativeClass("re.Match", { + constructor: function (match, re, str, pos, endpos) { + this.v = match; // javascript match object; + this.$match = new Sk.builtin.str(this.v[0]); + this.str = str; + this.$re = re; + this.$pos = pos; + this.$endpos = endpos; + // only calculate these if requested + this.$groupdict = null; + this.$groups = null; + this.$lastindex = null; + this.$lastgroup = null; + this.$regs = null; + }, + slots: { + tp$doc: "The result of re.match() and re.search().\nMatch objects always have a boolean value of True.", + $r() { + //e.g. + let ret = ""; + return new Sk.builtin.str(ret); + }, + tp$as_squence_or_mapping: true, + mp$subscript(item) { + const ret = this.get$group(item); + return ret === undefined ? Sk.builtin.none.none$ : new Sk.builtin.str(ret); + }, + }, + methods: { + group: { + $meth: function group(...gs) { + let ret; + if (gs.length <= 1) { + ret = this.get$group(gs[0]); + return ret === undefined ? Sk.builtin.none.none$ : new Sk.builtin.str(ret); + } + ret = []; + gs.forEach((g) => { + g = this.get$group(g); + ret.push(g === undefined ? Sk.builtin.none.none$ : new Sk.builtin.str(g)); + }); + return new Sk.builtin.tuple(ret); + }, + $flags: { MinArgs: 0 }, + $textsig: null, + $doc: + "group([group1, ...]) -> str or tuple.\n Return subgroup(s) of the match by indices or names.\n For 0 returns the entire match.", + }, + start: { + $meth: function start(g) { + const group = this.get$group(g); + if (group === undefined) { + return new Sk.builtin.int_(-1); + } + return new Sk.builtin.int_(this.str.v.indexOf(group, this.v.index + this.$pos)); + }, + $flags: { MinArgs: 0, MaxArgs: 1 }, + $textsig: "($self, group=0, /)", + $doc: "Return index of the start of the substring matched by group.", + }, + end: { + $meth: function end(g) { + const group = this.get$group(g); + if (group === undefined) { + return new Sk.builtin.int_(-1); + } + return new Sk.builtin.int_(this.str.v.indexOf(group, this.v.index + this.$pos) + [...group].length); + }, + $flags: { MinArgs: 0, MaxArgs: 1 }, + $textsig: "($self, group=0, /)", + $doc: "Return index of the end of the substring matched by group.", + }, + span: { + $meth: function span(g) { + return this.$span(g); + }, + $flags: { MinArgs: 0, MaxArgs: 1 }, + $textsig: "($self, group=0, /)", + $doc: "For match object m, return the 2-tuple (m.start(group), m.end(group)).", + }, + groups: { + $meth: function groups(d) { + if (this.$groups !== null) { + return this.$groups; + } + this.$groups = Array.from(this.v.slice(1), (x) => (x === undefined ? d : new Sk.builtin.str(x))); + this.$groups = new Sk.builtin.tuple(this.$groups); + return this.$groups; + }, + $flags: { NamedArgs: ["default"], Defaults: [Sk.builtin.none.none$] }, + $textsig: "($self, /, default=None)", + $doc: + "Return a tuple containing all the subgroups of the match, from 1.\n\n default\n Is used for groups that did not participate in the match.", + }, + groupdict: { + $meth: function groupdict(d) { + if (this.$groupdict !== null) { + return this.$groupdict; + } + if (this.v.groups === undefined) { + this.$groupdict = new Sk.builtin.dict(); + } else { + const arr = []; + Object.entries(this.v.groups).forEach(([name, val]) => { + arr.push(new Sk.builtin.str(name)); + arr.push(val === undefined ? d : new Sk.builtin.str(val)); + }); + this.$groupdict = new Sk.builtin.dict(arr); + } + return this.$groupdict; + }, + $flags: { NamedArgs: ["default"], Defaults: [Sk.builtin.none.none$] }, + $textsig: "($self, /, default=None)", + $doc: + "Return a dictionary containing all the named subgroups of the match, keyed by the subgroup name.\n\n default\n Is used for groups that did not participate in the match.", + }, + expand: { + $meth: function expand(template) { + if (!Sk.builtin.checkString(template)) { + throw new Sk.builtin.TypeError("expected str instance got " + Sk.abstr.typeName(template)); + } + template = template.toString(); + template = this.template$repl(template); + return new Sk.builtin.str(template); + }, + $flags: { OneArg: true }, + $textsig: "($self, /, template)", + $doc: "Return the string obtained by doing backslash substitution on the string template, as done by the sub() method.", + }, + __copy__: { + $meth: function __copy__() { + return this; + }, + $flags: { NoArgs: true }, + $textsig: "($self, /)", + $doc: null, + }, + __deepcopy__: { + $meth: function __deepcopy__() { + return this; + }, + $flags: { OneArg: true }, + $textsig: "($self, memo, /)", + $doc: null, + }, + }, + getsets: { + lastindex: { + $get() { + if (this.$lastindex !== null) { + return this.$lastindex; + } + let li = 0; + let lval; + this.v.forEach((val, i) => { + if (i && val !== undefined && lval !== val) { + li = i; + lval = val; + } + }); + this.$lastindex = li ? new Sk.builtin.int_(li) : Sk.builtin.none.none$; + return this.$lastindex; + }, + $doc: "The integer index of the last matched capturing group.", + }, + lastgroup: { + $get() { + if (this.$lastgroup !== null) { + return this.$lastgroup; + } + if (this.v.groups === undefined) { + this.$lastgroup = Sk.builtin.none.none$; + } else { + let lg; + Object.entries(this.v.groups).forEach(([name, val]) => { + if (val !== undefined) { + lg = name; + } + }); + this.$lastgroup = lg === undefined ? Sk.builtin.none.none$ : new Sk.builtin.str(lg); + } + return this.$lastgroup; + }, + $doc: "The name of the last matched capturing group.", + }, + regs: { + $get() { + if (this.$regs !== null) { + return this.$regs; + } + const arr = []; + this.v.forEach((x, i) => { + arr.push(this.$span(i)); + }); + this.$regs = new Sk.builtin.tuple(arr); + return this.$regs; + }, + }, + string: { + $get() { + return this.str; + }, + $doc: "The string passed to match() or search().", + }, + re: { + $get() { + return this.$re; + }, + $doc: "The regular expression object.", + }, + pos: { + $get() { + return new Sk.builtin.int_(this.$pos); + }, + $doc: "The index into the string at which the RE engine started looking for a match.", + }, + endpos: { + $get() { + return new Sk.builtin.int_(this.$endpos); + }, + $doc: "The index into the string beyond which the RE engine will not go.", + }, + }, + proto: { + get$group(g) { + if (g === undefined) { + return this.v[0]; + } else if (Sk.builtin.checkString(g)) { + g = g.toString(); + if (this.v.groups && Object.prototype.hasOwnProperty.call(this.v.groups, g)) { + return this.v.groups[g]; + } + } else if (Sk.misceval.isIndex(g)) { + g = Sk.misceval.asIndexSized(g); + if (g >= 0 && g < this.v.length) { + return this.v[g]; + } + } + throw new Sk.builtin.IndexError("no such group"); + }, + $span(g) { + const group = this.get$group(g); + if (group === undefined) { + return new Sk.builtin.tuple([new Sk.builtin.int_(-1), new Sk.builtin.int_(-1)]); + } + let idx; + if (group === "" && this.v[0] === "") { + idx = new Sk.builtin.int_(this.v.index); + return new Sk.builtin.tuple([idx, idx]); + } + idx = this.str.v.indexOf(group, this.v.index + this.$pos); + return new Sk.builtin.tuple([new Sk.builtin.int_(idx), new Sk.builtin.int_(idx + [...group].length)]); // want char length + }, + hasOwnProperty: Object.prototype.hasOwnProperty, + template$regex: /\\([1-9][0-9]|[1-9])|\\g<([1-9][0-9]*)>|\\g<([^\d\W]\w*)>|\\g?/g, + template$repl(template) { + return template.replace(this.template$regex, (match, idx, idxg, name, offset, orig) => { + let ret; + idx = idx || idxg; + if (idx !== undefined) { + ret = idx < this.v.length ? this.v[idx] || "" : undefined; + } else { + if (this.v.groups && this.hasOwnProperty.call(this.v.groups, name)) { + ret = this.v.groups[name] || ""; + } + } + if (ret === undefined) { + if (name) { + throw new Sk.builtin.IndexError("unknown group name '" + name + "'"); + } + throw new re.error("invalid group reference " + (idx || match.slice(2)) + " at position " + (offset + 1)); + } + return ret; + }); + }, + }, + flags: { + sk$acceptable_as_base_class: false, + }, + }); + + Sk.abstr.setUpModuleMethods("re", re, { + match: { + $meth: function match(pattern, string, flags) { + return _compile(pattern, flags).$match(string); + }, + $flags: { NamedArgs: ["pattern", "string", "flags"], Defaults: [zero] }, + $textsig: "($module, / , pattern, string, flags=0)", + $doc: "Try to apply the pattern at the start of the string, returning\n a Match object, or None if no match was found.", + }, + fullmatch: { + $meth: function fullmatch(pattern, string, flags) { + return _compile(pattern, flags).full$match(string); + }, + $flags: { NamedArgs: ["pattern", "string", "flags"], Defaults: [zero] }, + $textsig: "($module, / , pattern, string, flags=0)", + $doc: "Try to apply the pattern to all of the string, returning\n a Match object, or None if no match was found.", + }, + search: { + $meth: function search(pattern, string, flags) { + return _compile(pattern, flags).$search(string); + }, + $flags: { NamedArgs: ["pattern", "string", "flags"], Defaults: [zero] }, + $textsig: "($module, / , pattern, string, flags=0)", + $doc: "Scan through string looking for a match to the pattern, returning\n a Match object, or None if no match was found.", + }, + sub: { + $meth: function sub(pattern, repl, string, count, flags) { + return _compile(pattern, flags).$sub(repl, string, count); + }, + $flags: { NamedArgs: ["pattern", "repl", "string", "count", "flags"], Defaults: [zero, zero] }, + $textsig: "($module, / , pattern, string, count=0, flags=0)", + $doc: + "Return the string obtained by replacing the leftmost\n non-overlapping occurrences of the pattern in string by the\n replacement repl. repl can be either a string or a callable;\n if a string, backslash escapes in it are processed. If it is\n a callable, it's passed the Match object and must return\n a replacement string to be used.", + }, + subn: { + $meth: function subn(pattern, repl, string, count, flags) { + return _compile(pattern, flags).$subn(repl, string, count); + }, + $flags: { NamedArgs: ["pattern", "repl", "string", "count", "flags"], Defaults: [zero, zero] }, + $textsig: "($module, / , pattern, string, count=0, flags=0)", + $doc: + "Return a 2-tuple containing (new_string, number).\n new_string is the string obtained by replacing the leftmost\n non-overlapping occurrences of the pattern in the source\n string by the replacement repl. number is the number of\n substitutions that were made. repl can be either a string or a\n callable; if a string, backslash escapes in it are processed.\n If it is a callable, it's passed the Match object and must\n return a replacement string to be used.", + }, + split: { + $meth: function split(pattern, string, maxsplit, flags) { + return _compile(pattern, flags).$split(string, maxsplit); + }, + $flags: { NamedArgs: ["pattern", "string", "maxsplit", "flags"], Defaults: [zero, zero] }, + $textsig: "($module, / , pattern, string, maxsplit=0, flags=0)", + $doc: + "Split the source string by the occurrences of the pattern,\n returning a list containing the resulting substrings. If\n capturing parentheses are used in pattern, then the text of all\n groups in the pattern are also returned as part of the resulting\n list. If maxsplit is nonzero, at most maxsplit splits occur,\n and the remainder of the string is returned as the final element\n of the list.", + }, + findall: { + $meth: function findall(pattern, string, flags) { + return _compile(pattern, flags).find$all(string); + }, + $flags: { NamedArgs: ["pattern", "string", "flags"], Defaults: [zero] }, + $textsig: "($module, / , pattern, string, flags=0)", + $doc: + "Return a list of all non-overlapping matches in the string.\n\n If one or more capturing groups are present in the pattern, return\n a list of groups; this will be a list of tuples if the pattern\n has more than one group.\n\n Empty matches are included in the result.", + }, + finditer: { + $meth: function finditer(pattern, string, flags) { + return _compile(pattern, flags).find$iter(string); + }, + $flags: { NamedArgs: ["pattern", "string", "flags"], Defaults: [zero] }, + $textsig: "($module, / , pattern, string, flags=0)", + $doc: + "Return an iterator over all non-overlapping matches in the\n string. For each match, the iterator returns a Match object.\n\n Empty matches are included in the result.", + }, + compile: { + $meth: function compile(pattern, flags) { + return _compile(pattern, flags); + }, + $flags: { NamedArgs: ["pattern", "flags"], Defaults: [zero] }, + $textsig: "($module, / , pattern, flags=0)", + $doc: "Compile a regular expression pattern, returning a Pattern object.", + }, + purge: { + $meth: function purge() { + Object.keys(_compiled_patterns).forEach((key) => { + delete _compiled_patterns[key]; + }); + return Sk.builtin.none.none$; + }, + $flags: { NoArgs: true }, + $textsig: "($module, / )", + $doc: "Clear the regular expression caches", + }, + template: { + $meth: function template(pattern, flags) { + return _compile(pattern, Sk.abstr.numberBinOp(re.T, flags, "BitOr")); + }, + $flags: { NamedArgs: ["pattern", "flags"], Defaults: [zero] }, + $textsig: "($module, / , pattern, flags=0)", + $doc: "Compile a template pattern, returning a Pattern object", + }, + escape: { + $meth: function (pattern) { + if (!Sk.builtin.checkString(pattern)) { + throw new Sk.builtin.TypeError("expected a str instances, got " + Sk.abstr.typeName(pattern)); + } + pattern = pattern.toString(); + pattern = pattern.replace(escape_chrs, "\\$&"); + return new Sk.builtin.str(pattern); + }, + $flags: { NamedArgs: ["pattern"], Defaults: [] }, + $textsig: "($module, / , pattern)", + $doc: "\n Escape special characters in a string.\n ", + }, + }); + const escape_chrs = /[\&\~\#.*+\-?^${}()|[\]\\\t\r\v\f\n ]/g; - return mod; -}; + return re; +} From b55de7756e79931783b6c7e3a37afa326dec2246 Mon Sep 17 00:00:00 2001 From: mrcork Date: Mon, 5 Oct 2020 14:25:34 +0800 Subject: [PATCH 112/137] re tests --- test/run/t460.py | 2 +- test/run/t460.py.real | 1 - test/unit3/test_re.py | 2360 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 2355 insertions(+), 8 deletions(-) diff --git a/test/run/t460.py b/test/run/t460.py index eb0384ace0..ffb91e8885 100644 --- a/test/run/t460.py +++ b/test/run/t460.py @@ -5,7 +5,7 @@ print re.split("\W+", "Words, words, words.", 1) print re.split('[a-f]+', '0a3B9', 0, re.IGNORECASE) print re.split("(\W+)", '...words, words...') -print re.split('x*', 'foo') +# print re.split('x*', 'foo') # different in py3 #print re.split("(?m)^$", "foo\n\nbar\n") print re.findall('\w+', "Words, words, words.") diff --git a/test/run/t460.py.real b/test/run/t460.py.real index 521ace89b8..4480a3a1da 100644 --- a/test/run/t460.py.real +++ b/test/run/t460.py.real @@ -3,7 +3,6 @@ ['Words', 'words, words.'] ['0', '3', '9'] ['', '...', 'words', ', ', 'words', '...', ''] -['foo'] ['Words', 'words', 'words'] [('abc', 'def')] [('abc', 'def'), ('abc', 'def'), ('abc', 'def')] diff --git a/test/unit3/test_re.py b/test/unit3/test_re.py index 89a4d48a83..d3ddeced20 100644 --- a/test/unit3/test_re.py +++ b/test/unit3/test_re.py @@ -1,8 +1,2357 @@ -""" Unit test for re module""" -import unittest +# from test.support import (gc_collect, bigmemtest, _2G, +# cpython_only, captured_stdout) +# import locale import re +# import sre_compile +import string +import unittest +# import warnings +# from re import Scanner +# from weakref import proxy + +# Misc tests from Tim Peters' re.doc + +# WARNING: Don't change details in these tests if you don't know +# what you're doing. Some of these tests were carefully modeled to +# cover most of the code. + +class S(str): + def __getitem__(self, index): + return S(super().__getitem__(index)) + +class B(bytes): + def __getitem__(self, index): + return B(super().__getitem__(index)) class ReTests(unittest.TestCase): + + def assertTypedEqual(self, actual, expect, msg=None): + self.assertEqual(actual, expect, msg) + def recurse(actual, expect): + if isinstance(expect, (tuple, list)): + for x, y in zip(actual, expect): + recurse(x, y) + else: + self.assertIs(type(actual), type(expect), msg) + recurse(actual, expect) + + def checkPatternError(self, pattern, errmsg, pos=None): + with self.assertRaises(re.error) as cm: + re.compile(pattern) + # with self.subTest(pattern=pattern): + # err = cm.exception + # self.assertEqual(err.msg, errmsg) + # if pos is not None: + # self.assertEqual(err.pos, pos) + + def checkTemplateError(self, pattern, repl, string, errmsg, pos=None): + with self.assertRaises(re.error) as cm: + re.sub(pattern, repl, string) + # with self.subTest(pattern=pattern, repl=repl): + # err = cm.exception + # self.assertEqual(err.msg, errmsg) + # if pos is not None: + # self.assertEqual(err.pos, pos) + + # def test_keep_buffer(self): + # # See bug 14212 + # b = bytearray(b'x') + # it = re.finditer(b'a', b) + # with self.assertRaises(BufferError): + # b.extend(b'x'*400) + # list(it) + # del it + # gc_collect() + # b.extend(b'x'*400) + + # def test_weakref(self): + # s = 'QabbbcR' + # x = re.compile('ab+c') + # y = proxy(x) + # self.assertEqual(x.findall('QabbbcR'), y.findall('QabbbcR')) + + def test_search_star_plus(self): + self.assertEqual(re.search('x*', 'axx').span(0), (0, 0)) + self.assertEqual(re.search('x*', 'axx').span(), (0, 0)) + self.assertEqual(re.search('x+', 'axx').span(0), (1, 3)) + self.assertEqual(re.search('x+', 'axx').span(), (1, 3)) + self.assertIsNone(re.search('x', 'aaa')) + self.assertEqual(re.match('a*', 'xxx').span(0), (0, 0)) + self.assertEqual(re.match('a*', 'xxx').span(), (0, 0)) + self.assertEqual(re.match('x*', 'xxxa').span(0), (0, 3)) + self.assertEqual(re.match('x*', 'xxxa').span(), (0, 3)) + self.assertIsNone(re.match('a+', 'xxx')) + + def bump_num(self, matchobj): + int_value = int(matchobj.group(0)) + return str(int_value + 1) + + def test_basic_re_sub(self): + self.assertTypedEqual(re.sub('y', 'a', 'xyz'), 'xaz') + self.assertTypedEqual(re.sub('y', S('a'), S('xyz')), 'xaz') + # self.assertTypedEqual(re.sub(b'y', b'a', b'xyz'), b'xaz') + # self.assertTypedEqual(re.sub(b'y', B(b'a'), B(b'xyz')), b'xaz') + # self.assertTypedEqual(re.sub(b'y', bytearray(b'a'), bytearray(b'xyz')), b'xaz') + # self.assertTypedEqual(re.sub(b'y', memoryview(b'a'), memoryview(b'xyz')), b'xaz') + for y in ("\xe0", "\u0430", "\U0001d49c"): + self.assertEqual(re.sub(y, 'a', 'x%sz' % y), 'xaz') + + self.assertEqual(re.sub("(?i)b+", "x", "bbbb BBBB"), 'x x') + self.assertEqual(re.sub(r'\d+', self.bump_num, '08.2 -2 23x99y'), + '9.3 -3 24x100y') + self.assertEqual(re.sub(r'\d+', self.bump_num, '08.2 -2 23x99y', 3), + '9.3 -3 23x99y') + self.assertEqual(re.sub(r'\d+', self.bump_num, '08.2 -2 23x99y', count=3), + '9.3 -3 23x99y') + + self.assertEqual(re.sub('.', lambda m: r"\n", 'x'), '\\n') + # self.assertEqual(re.sub('.', r"\n", 'x'), '\n') + + s = r"\1\1" + self.assertEqual(re.sub('(.)', s, 'x'), 'xx') + # self.assertEqual(re.sub('(.)', s.replace('\\', r'\\'), 'x'), s) + self.assertEqual(re.sub('(.)', lambda m: s, 'x'), s) + + self.assertEqual(re.sub('(?Px)', r'\g\g', 'xx'), 'xxxx') + self.assertEqual(re.sub('(?Px)', r'\g\g<1>', 'xx'), 'xxxx') + self.assertEqual(re.sub('(?Px)', r'\g\g', 'xx'), 'xxxx') + self.assertEqual(re.sub('(?Px)', r'\g<1>\g<1>', 'xx'), 'xxxx') + + # @TODO escape characters break this + # self.assertEqual(re.sub('a', r'\t\n\v\r\f\a\b', 'a'), '\t\n\v\r\f\a\b') + self.assertEqual(re.sub('a', '\t\n\v\r\f\b', 'a'), '\t\n\v\r\f\b') + self.assertEqual(re.sub('a', '\t\n\v\r\f\b', 'a'), + (chr(9)+chr(10)+chr(11)+chr(13)+chr(12)+chr(8))) + # self.assertEqual(re.sub('a', '\t\n\v\r\f\a\b', 'a'), + # (chr(9)+chr(10)+chr(11)+chr(13)+chr(12)+chr(7)+chr(8))) + + # for c in 'cdehijklmopqsuwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ': + # # with self.subTest(c): + # with self.assertRaises(re.error): + # self.assertEqual(re.sub('a', '\\' + c, 'a'), '\\' + c) + + self.assertEqual(re.sub(r'^\s*', 'X', 'test'), 'Xtest') + + def test_bug_449964(self): + # fails for group followed by other escape + self.assertEqual(re.sub(r'(?Px)', '\g<1>\g<1>\b', 'xx'), + 'xx\bxx\b') + # skulpt - the raw string doesn't work here + # self.assertEqual(re.sub(r'(?Px)', r'\g<1>\g<1>\b', 'xx'), + # 'xx\bxx\b') + + def test_bug_449000(self): + pass + # Test for sub() on escaped characters + + # escaped characters like this don't really work! + # self.assertEqual(re.sub(r'\r\n', r'\n', 'abc\r\ndef\r\n'), + # 'abc\ndef\n') + # self.assertEqual(re.sub('\r\n', r'\n', 'abc\r\ndef\r\n'), + # 'abc\ndef\n') + # self.assertEqual(re.sub(r'\r\n', '\n', 'abc\r\ndef\r\n'), + # 'abc\ndef\n') + # self.assertEqual(re.sub('\r\n', '\n', 'abc\r\ndef\r\n'), + # 'abc\ndef\n') + + def test_bug_1661(self): + # Verify that flags do not get silently ignored with compiled patterns + pattern = re.compile('.') + self.assertRaises(ValueError, re.match, pattern, 'A', re.I) + self.assertRaises(ValueError, re.search, pattern, 'A', re.I) + self.assertRaises(ValueError, re.findall, pattern, 'A', re.I) + self.assertRaises(ValueError, re.compile, pattern, re.I) + + # def test_bug_3629(self): + # #@TODO not supported this type of regex + # # A regex that triggered a bug in the sre-code validator + # re.compile("(?P)(?(quote))") + + def test_sub_template_numeric_escape(self): + # bug 776311 and friends + # self.assertEqual(re.sub('x', r'\0', 'x'), '\0') + # self.assertEqual(re.sub('x', r'\000', 'x'), '\000') + # self.assertEqual(re.sub('x', r'\001', 'x'), '\001') + # self.assertEqual(re.sub('x', r'\008', 'x'), '\0' + '8') + # self.assertEqual(re.sub('x', r'\009', 'x'), '\0' + '9') + # self.assertEqual(re.sub('x', r'\111', 'x'), '\111') + # self.assertEqual(re.sub('x', r'\117', 'x'), '\117') + # self.assertEqual(re.sub('x', r'\377', 'x'), '\377') + + # self.assertEqual(re.sub('x', r'\1111', 'x'), '\1111') + # self.assertEqual(re.sub('x', r'\1111', 'x'), '\111' + '1') + + # self.assertEqual(re.sub('x', r'\00', 'x'), '\x00') + # self.assertEqual(re.sub('x', r'\07', 'x'), '\x07') + # self.assertEqual(re.sub('x', r'\08', 'x'), '\0' + '8') + # self.assertEqual(re.sub('x', r'\09', 'x'), '\0' + '9') + # self.assertEqual(re.sub('x', r'\0a', 'x'), '\0' + 'a') + + # self.checkTemplateError('x', r'\400', 'x', + # r'octal escape value \400 outside of ' + # r'range 0-0o377', 0) + # self.checkTemplateError('x', r'\777', 'x', + # r'octal escape value \777 outside of ' + # r'range 0-0o377', 0) + + self.checkTemplateError('x', r'\1', 'x', 'invalid group reference 1', 1) + self.checkTemplateError('x', r'\8', 'x', 'invalid group reference 8', 1) + self.checkTemplateError('x', r'\9', 'x', 'invalid group reference 9', 1) + self.checkTemplateError('x', r'\11', 'x', 'invalid group reference 11', 1) + self.checkTemplateError('x', r'\18', 'x', 'invalid group reference 18', 1) + self.checkTemplateError('x', r'\1a', 'x', 'invalid group reference 1', 1) + self.checkTemplateError('x', r'\90', 'x', 'invalid group reference 90', 1) + self.checkTemplateError('x', r'\99', 'x', 'invalid group reference 99', 1) + self.checkTemplateError('x', r'\118', 'x', 'invalid group reference 11', 1) + self.checkTemplateError('x', r'\11a', 'x', 'invalid group reference 11', 1) + self.checkTemplateError('x', r'\181', 'x', 'invalid group reference 18', 1) + self.checkTemplateError('x', r'\800', 'x', 'invalid group reference 80', 1) + # self.checkTemplateError('x', r'\8', '', 'invalid group reference 8', 1) + + # in python2.3 (etc), these loop endlessly in sre_parser.py + self.assertEqual(re.sub('(((((((((((x)))))))))))', r'\11', 'x'), 'x') + self.assertEqual(re.sub('((((((((((y))))))))))(.)', r'\118', 'xyz'), + 'xz8') + self.assertEqual(re.sub('((((((((((y))))))))))(.)', r'\11a', 'xyz'), + 'xza') + + def test_qualified_re_sub(self): + self.assertEqual(re.sub('a', 'b', 'aaaaa'), 'bbbbb') + self.assertEqual(re.sub('a', 'b', 'aaaaa', 1), 'baaaa') + self.assertEqual(re.sub('a', 'b', 'aaaaa', count=1), 'baaaa') + + def test_bug_114660(self): + self.assertEqual(re.sub(r'(\S)\s+(\S)', r'\1 \2', 'hello there'), + 'hello there') + + def test_symbolic_groups(self): + # re.compile(r'(?Px)(?P=a)(?(a)y)') + # re.compile(r'(?Px)(?P=a1)(?(a1)y)') + # re.compile(r'(?Px)\1(?(1)y)') + self.checkPatternError(r'(?P)(?P)', + "redefinition of group name 'a' as group 2; " + "was group 1") + # self.checkPatternError(r'(?P(?P=a))', + # "cannot refer to an open group", 10) + self.checkPatternError(r'(?Pxy)', 'unknown extension ?Px') + self.checkPatternError(r'(?P)(?P=a', 'missing ), unterminated name', 11) + self.checkPatternError(r'(?P=', 'missing group name', 4) + self.checkPatternError(r'(?P=)', 'missing group name', 4) + self.checkPatternError(r'(?P=1)', "bad character in group name '1'", 4) + self.checkPatternError(r'(?P=a)', "unknown group name 'a'") + self.checkPatternError(r'(?P=a1)', "unknown group name 'a1'") + self.checkPatternError(r'(?P=a.)', "bad character in group name 'a.'", 4) + self.checkPatternError(r'(?P<)', 'missing >, unterminated name', 4) + self.checkPatternError(r'(?P, unterminated name', 4) + self.checkPatternError(r'(?P<', 'missing group name', 4) + self.checkPatternError(r'(?P<>)', 'missing group name', 4) + self.checkPatternError(r'(?P<1>)', "bad character in group name '1'", 4) + self.checkPatternError(r'(?P)', "bad character in group name 'a.'", 4) + self.checkPatternError(r'(?(', 'missing group name', 3) + self.checkPatternError(r'(?())', 'missing group name', 3) + self.checkPatternError(r'(?(a))', "unknown group name 'a'", 3) + self.checkPatternError(r'(?(-1))', "bad character in group name '-1'", 3) + self.checkPatternError(r'(?(1a))', "bad character in group name '1a'", 3) + self.checkPatternError(r'(?(a.))', "bad character in group name 'a.'", 3) + # New valid/invalid identifiers in Python 3 + # re.compile('(?P<µ>x)(?P=µ)(?(µ)y)') + # re.compile('(?P<𝔘𝔫𝔦𝔠𝔬𝔡𝔢>x)(?P=𝔘𝔫𝔦𝔠𝔬𝔡𝔢)(?(𝔘𝔫𝔦𝔠𝔬𝔡𝔢)y)') + self.checkPatternError('(?P<©>x)', "bad character in group name '©'", 4) + # Support > 100 groups. + # pat = '|'.join('x(?P%x)y' % (i, i) for i in range(1, 200 + 1)) + # pat = '(?:%s)(?(200)z|t)' % pat + # self.assertEqual(re.match(pat, 'xc8yz').span(), (0, 5)) + + def test_symbolic_refs(self): + self.checkTemplateError('(?Px)', r'\g, unterminated name', 3) + self.checkTemplateError('(?Px)', r'\g<', 'xx', + 'missing group name', 3) + self.checkTemplateError('(?Px)', r'\g', 'xx', 'missing <', 2) + self.checkTemplateError('(?Px)', r'\g', 'xx', + "bad character in group name 'a a'", 3) + self.checkTemplateError('(?Px)', r'\g<>', 'xx', + 'missing group name', 3) + self.checkTemplateError('(?Px)', r'\g<1a1>', 'xx', + "bad character in group name '1a1'", 3) + self.checkTemplateError('(?Px)', r'\g<2>', 'xx', + 'invalid group reference 2', 3) + self.checkTemplateError('(?Px)', r'\2', 'xx', + 'invalid group reference 2', 1) + with self.assertRaises(IndexError): + re.sub('(?Px)', r'\g', 'xx') + self.assertEqual(re.sub('(?Px)|(?Py)', r'\g', 'xx'), '') + self.assertEqual(re.sub('(?Px)|(?Py)', r'\2', 'xx'), '') + self.checkTemplateError('(?Px)', r'\g<-1>', 'xx', + "bad character in group name '-1'", 3) + # New valid/invalid identifiers in Python 3 + # self.assertEqual(re.sub('(?P<µ>x)', r'\g<µ>', 'xx'), 'xx') + # self.assertEqual(re.sub('(?P<𝔘𝔫𝔦𝔠𝔬𝔡𝔢>x)', r'\g<𝔘𝔫𝔦𝔠𝔬𝔡𝔢>', 'xx'), 'xx') + self.checkTemplateError('(?Px)', r'\g<©>', 'xx', + "bad character in group name '©'", 3) + # Support > 100 groups. + pat = '|'.join('x(?P%x)y' % (i, i) for i in range(1, 200 + 1)) + self.assertEqual(re.sub(pat, r'\g<200>', 'xc8yzxc8y'), 'c8zc8') + + def test_re_subn(self): + self.assertEqual(re.subn("(?i)b+", "x", "bbbb BBBB"), ('x x', 2)) + self.assertEqual(re.subn("b+", "x", "bbbb BBBB"), ('x BBBB', 1)) + self.assertEqual(re.subn("b+", "x", "xyz"), ('xyz', 0)) + self.assertEqual(re.subn("b*", "x", "xyz"), ('xxxyxzx', 4)) + self.assertEqual(re.subn("b*", "x", "xyz", 2), ('xxxyz', 2)) + self.assertEqual(re.subn("b*", "x", "xyz", count=2), ('xxxyz', 2)) + + def test_re_split(self): + for string in ":a:b::c", S(":a:b::c"): + self.assertTypedEqual(re.split(":", string), + ['', 'a', 'b', '', 'c']) + self.assertTypedEqual(re.split(":+", string), + ['', 'a', 'b', 'c']) + self.assertTypedEqual(re.split("(:+)", string), + ['', ':', 'a', ':', 'b', '::', 'c']) + # for string in (b":a:b::c", B(b":a:b::c"), bytearray(b":a:b::c"), + # memoryview(b":a:b::c")): + # self.assertTypedEqual(re.split(b":", string), + # [b'', b'a', b'b', b'', b'c']) + # self.assertTypedEqual(re.split(b":+", string), + # [b'', b'a', b'b', b'c']) + # self.assertTypedEqual(re.split(b"(:+)", string), + # [b'', b':', b'a', b':', b'b', b'::', b'c']) + for a, b, c in ("\xe0\xdf\xe7", "\u0430\u0431\u0432", + "\U0001d49c\U0001d49e\U0001d4b5"): + string = ":%s:%s::%s" % (a, b, c) + self.assertEqual(re.split(":", string), ['', a, b, '', c]) + self.assertEqual(re.split(":+", string), ['', a, b, c]) + self.assertEqual(re.split("(:+)", string), + ['', ':', a, ':', b, '::', c]) + + self.assertEqual(re.split("(?::+)", ":a:b::c"), ['', 'a', 'b', 'c']) + self.assertEqual(re.split("(:)+", ":a:b::c"), + ['', ':', 'a', ':', 'b', ':', 'c']) + self.assertEqual(re.split("([b:]+)", ":a:b::c"), + ['', ':', 'a', ':b::', 'c']) + self.assertEqual(re.split("(b)|(:+)", ":a:b::c"), + ['', None, ':', 'a', None, ':', '', 'b', None, '', + None, '::', 'c']) + self.assertEqual(re.split("(?:b)|(?::+)", ":a:b::c"), + ['', 'a', '', '', 'c']) + + # python is weird here - skulpt behaves well enough and more like python 2 + for sep, expected in [ + (':*', ['', '', 'a', '', 'b', '', 'c', '']), + ('(?::*)', ['', '', 'a', '', 'b', '', 'c', '']), + ('(:*)', ['', ':', '', '', 'a', ':', '', '', 'b', '::', '', '', 'c', '', '']), + ('(:)*', ['', ':', '', None, 'a', ':', '', None, 'b', ':', '', None, 'c', None, '']), + ]: + # with self.subTest(sep=sep): + self.assertTypedEqual(re.split(sep, ':a:b::c'), expected) + + for sep, expected in [ + # ('', ['', ':', 'a', ':', 'b', ':', ':', 'c', '']), + # (r'\b', [':', 'a', ':', 'b', '::', 'c', '']), + # (r'(?=:)', ['', ':a', ':b', ':', ':c']), + (r'(?<=:)', [':', 'a:', 'b:', ':', 'c']), + ]: + # with self.subTest(sep=sep): + self.assertTypedEqual(re.split(sep, ':a:b::c'), expected) + + def test_qualified_re_split(self): + self.assertEqual(re.split(":", ":a:b::c", 2), ['', 'a', 'b::c']) + self.assertEqual(re.split(":", ":a:b::c", maxsplit=2), ['', 'a', 'b::c']) + self.assertEqual(re.split(':', 'a:b:c:d', maxsplit=2), ['a', 'b', 'c:d']) + self.assertEqual(re.split("(:)", ":a:b::c", maxsplit=2), + ['', ':', 'a', ':', 'b::c']) + self.assertEqual(re.split("(:+)", ":a:b::c", maxsplit=2), + ['', ':', 'a', ':', 'b::c']) + self.assertEqual(re.split("(:*)", ":a:b::c", maxsplit=2), + ['', ':', '', '', 'a:b::c']) + + def test_re_findall(self): + self.assertEqual(re.findall(":+", "abc"), []) + for string in "a:b::c:::d", S("a:b::c:::d"): + self.assertTypedEqual(re.findall(":+", string), + [":", "::", ":::"]) + self.assertTypedEqual(re.findall("(:+)", string), + [":", "::", ":::"]) + self.assertTypedEqual(re.findall("(:)(:*)", string), + [(":", ""), (":", ":"), (":", "::")]) + # for string in (b"a:b::c:::d", B(b"a:b::c:::d"), bytearray(b"a:b::c:::d"), + # memoryview(b"a:b::c:::d")): + # self.assertTypedEqual(re.findall(b":+", string), + # [b":", b"::", b":::"]) + # self.assertTypedEqual(re.findall(b"(:+)", string), + # [b":", b"::", b":::"]) + # self.assertTypedEqual(re.findall(b"(:)(:*)", string), + # [(b":", b""), (b":", b":"), (b":", b"::")]) + for x in ("\xe0", "\u0430", "\U0001d49c"): + xx = x * 2 + xxx = x * 3 + string = "a%sb%sc%sd" % (x, xx, xxx) + self.assertEqual(re.findall("%s+" % x, string), [x, xx, xxx]) + self.assertEqual(re.findall("(%s+)" % x, string), [x, xx, xxx]) + self.assertEqual(re.findall("(%s)(%s*)" % (x, x), string), + [(x, ""), (x, x), (x, xx)]) + + def test_bug_117612(self): + self.assertEqual(re.findall(r"(a|(b))", "aba"), + [("a", ""),("b", "b"),("a", "")]) + + def test_re_match(self): + for string in 'a', S('a'): + self.assertEqual(re.match('a', string).groups(), ()) + self.assertEqual(re.match('(a)', string).groups(), ('a',)) + self.assertEqual(re.match('(a)', string).group(0), 'a') + self.assertEqual(re.match('(a)', string).group(1), 'a') + self.assertEqual(re.match('(a)', string).group(1, 1), ('a', 'a')) + # for string in b'a', B(b'a'), bytearray(b'a'), memoryview(b'a'): + # self.assertEqual(re.match(b'a', string).groups(), ()) + # self.assertEqual(re.match(b'(a)', string).groups(), (b'a',)) + # self.assertEqual(re.match(b'(a)', string).group(0), b'a') + # self.assertEqual(re.match(b'(a)', string).group(1), b'a') + # self.assertEqual(re.match(b'(a)', string).group(1, 1), (b'a', b'a')) + for a in ("\xe0", "\u0430", "\U0001d49c"): + self.assertEqual(re.match(a, a).groups(), ()) + self.assertEqual(re.match('(%s)' % a, a).groups(), (a,)) + self.assertEqual(re.match('(%s)' % a, a).group(0), a) + self.assertEqual(re.match('(%s)' % a, a).group(1), a) + self.assertEqual(re.match('(%s)' % a, a).group(1, 1), (a, a)) + + pat = re.compile('((a)|(b))(c)?') + self.assertEqual(pat.match('a').groups(), ('a', 'a', None, None)) + self.assertEqual(pat.match('b').groups(), ('b', None, 'b', None)) + self.assertEqual(pat.match('ac').groups(), ('a', 'a', None, 'c')) + self.assertEqual(pat.match('bc').groups(), ('b', None, 'b', 'c')) + self.assertEqual(pat.match('bc').groups(""), ('b', "", 'b', 'c')) + + pat = re.compile('(?:(?Pa)|(?Pb))(?Pc)?') + self.assertEqual(pat.match('a').group(1, 2, 3), ('a', None, None)) + self.assertEqual(pat.match('b').group('a1', 'b2', 'c3'), + (None, 'b', None)) + self.assertEqual(pat.match('ac').group(1, 'b2', 3), ('a', None, 'c')) + + def test_group(self): + class Index: + def __init__(self, value): + self.value = value + def __index__(self): + return self.value + # A single group + m = re.match('(a)(b)', 'ab') + self.assertEqual(m.group(), 'ab') + self.assertEqual(m.group(0), 'ab') + self.assertEqual(m.group(1), 'a') + self.assertEqual(m.group(Index(1)), 'a') + self.assertRaises(IndexError, m.group, -1) + self.assertRaises(IndexError, m.group, 3) + self.assertRaises(IndexError, m.group, 1<<1000) + self.assertRaises(IndexError, m.group, Index(1<<1000)) + self.assertRaises(IndexError, m.group, 'x') + # Multiple groups + self.assertEqual(m.group(2, 1), ('b', 'a')) + self.assertEqual(m.group(Index(2), Index(1)), ('b', 'a')) + + def test_match_getitem(self): + pat = re.compile('(?:(?Pa)|(?Pb))(?Pc)?') + + m = pat.match('a') + self.assertEqual(m['a1'], 'a') + self.assertEqual(m['b2'], None) + self.assertEqual(m['c3'], None) + # self.assertEqual('a1={a1} b2={b2} c3={c3}'.format_map(m), 'a1=a b2=None c3=None') + self.assertEqual(m[0], 'a') + self.assertEqual(m[1], 'a') + self.assertEqual(m[2], None) + self.assertEqual(m[3], None) + with self.assertRaises(IndexError): + m['X'] + with self.assertRaises(IndexError): + m[-1] + with self.assertRaises(IndexError): + m[4] + with self.assertRaises(IndexError): + m[0, 1] + with self.assertRaises(IndexError): + m[(0,)] + with self.assertRaises(IndexError): + m[(0, 1)] + # with self.assertRaisesRegex(IndexError, 'no such group'): + # 'a1={a2}'.format_map(m) + + m = pat.match('ac') + self.assertEqual(m['a1'], 'a') + self.assertEqual(m['b2'], None) + self.assertEqual(m['c3'], 'c') + # self.assertEqual('a1={a1} b2={b2} c3={c3}'.format_map(m), 'a1=a b2=None c3=c') + self.assertEqual(m[0], 'ac') + self.assertEqual(m[1], 'a') + self.assertEqual(m[2], None) + self.assertEqual(m[3], 'c') + + # Cannot assign. + with self.assertRaises(TypeError): + m[0] = 1 + + # No len(). + self.assertRaises(TypeError, len, m) + + def test_re_fullmatch(self): + # Issue 16203: Proposal: add re.fullmatch() method. + self.assertEqual(re.fullmatch(r"a", "a").span(), (0, 1)) + for string in "ab", S("ab"): + self.assertEqual(re.fullmatch(r"a|ab", string).span(), (0, 2)) + # for string in b"ab", B(b"ab"), bytearray(b"ab"), memoryview(b"ab"): + # self.assertEqual(re.fullmatch(br"a|ab", string).span(), (0, 2)) + for a, b in "\xe0\xdf", "\u0430\u0431", "\U0001d49c\U0001d49e": + r = r"%s|%s" % (a, a + b) + self.assertEqual(re.fullmatch(r, a + b).span(), (0, 2)) + self.assertEqual(re.fullmatch(r".*?$", "abc").span(), (0, 3)) + self.assertEqual(re.fullmatch(r".*?", "abc").span(), (0, 3)) + self.assertEqual(re.fullmatch(r"a.*?b", "ab").span(), (0, 2)) + self.assertEqual(re.fullmatch(r"a.*?b", "abb").span(), (0, 3)) + self.assertEqual(re.fullmatch(r"a.*?b", "axxb").span(), (0, 4)) + self.assertIsNone(re.fullmatch(r"a+", "ab")) + self.assertIsNone(re.fullmatch(r"abc$", "abc\n")) + self.assertIsNone(re.fullmatch(r"abc\Z", "abc\n")) + self.assertIsNone(re.fullmatch(r"(?m)abc$", "abc\n")) + self.assertEqual(re.fullmatch(r"ab(?=c)cd", "abcd").span(), (0, 4)) + self.assertEqual(re.fullmatch(r"ab(?<=b)cd", "abcd").span(), (0, 4)) + self.assertEqual(re.fullmatch(r"(?=a|ab)ab", "ab").span(), (0, 2)) + + self.assertEqual( + re.compile(r"bc").fullmatch("abcd", pos=1, endpos=3).span(), (1, 3)) + self.assertEqual( + re.compile(r".*?$").fullmatch("abcd", pos=1, endpos=3).span(), (1, 3)) + self.assertEqual( + re.compile(r".*?").fullmatch("abcd", pos=1, endpos=3).span(), (1, 3)) + + def test_re_groupref_exists(self): + pass + # Not supported in Skulpt + # self.assertEqual(re.match(r'^(\()?([^()]+)(?(1)\))$', '(a)').groups(), + # ('(', 'a')) + # self.assertEqual(re.match(r'^(\()?([^()]+)(?(1)\))$', 'a').groups(), + # (None, 'a')) + # self.assertIsNone(re.match(r'^(\()?([^()]+)(?(1)\))$', 'a)')) + # self.assertIsNone(re.match(r'^(\()?([^()]+)(?(1)\))$', '(a')) + # self.assertEqual(re.match('^(?:(a)|c)((?(1)b|d))$', 'ab').groups(), + # ('a', 'b')) + # self.assertEqual(re.match(r'^(?:(a)|c)((?(1)b|d))$', 'cd').groups(), + # (None, 'd')) + # self.assertEqual(re.match(r'^(?:(a)|c)((?(1)|d))$', 'cd').groups(), + # (None, 'd')) + # self.assertEqual(re.match(r'^(?:(a)|c)((?(1)|d))$', 'a').groups(), + # ('a', '')) + + # Tests for bug #1177831: exercise groups other than the first group + # p = re.compile('(?Pa)(?Pb)?((?(g2)c|d))') + # self.assertEqual(p.match('abc').groups(), + # ('a', 'b', 'c')) + # self.assertEqual(p.match('ad').groups(), + # ('a', None, 'd')) + # self.assertIsNone(p.match('abd')) + # self.assertIsNone(p.match('ac')) + + # Support > 100 groups. + # pat = '|'.join('x(?P%x)y' % (i, i) for i in range(1, 200 + 1)) + # pat = '(?:%s)(?(200)z)' % pat + # self.assertEqual(re.match(pat, 'xc8yz').span(), (0, 5)) + + # self.checkPatternError(r'(?P)(?(0))', 'bad group number', 10) + # self.checkPatternError(r'()(?(1)a|b', + # 'missing ), unterminated subpattern', 2) + # self.checkPatternError(r'()(?(1)a|b|c)', + # 'conditional backref with more than ' + # 'two branches', 10) + + # def test_re_groupref_overflow(self): + # from sre_constants import MAXGROUPS + # self.checkTemplateError('()', r'\g<%s>' % MAXGROUPS, 'xx', + # 'invalid group reference %d' % MAXGROUPS, 3) + # self.checkPatternError(r'(?P)(?(%d))' % MAXGROUPS, + # 'invalid group reference %d' % MAXGROUPS, 10) + + def test_re_groupref(self): + self.assertEqual(re.match(r'^(\|)?([^()]+)\1$', '|a|').groups(), + ('|', 'a')) + self.assertEqual(re.match(r'^(\|)?([^()]+)\1?$', 'a').groups(), + (None, 'a')) + # the following two are matches in skulpt and in javascritpt + # self.assertIsNone(re.match(r'^(\|)?([^()]+)\1$', 'a|')) + # self.assertIsNone(re.match(r'^(\|)?([^()]+)\1$', '|a')) + self.assertEqual(re.match(r'^(?:(a)|c)(\1)$', 'aa').groups(), + ('a', 'a')) + self.assertEqual(re.match(r'^(?:(a)|c)(\1)?$', 'c').groups(), + (None, None)) + + # self.checkPatternError(r'(abc\1)', 'cannot refer to an open group', 4) + + def test_groupdict(self): + self.assertEqual(re.match('(?Pfirst) (?Psecond)', + 'first second').groupdict(), + {'first':'first', 'second':'second'}) + + def test_expand(self): + self.assertEqual(re.match("(?Pfirst) (?Psecond)", + "first second") + .expand(r"\2 \1 \g \g"), + "second first second first") + self.assertEqual(re.match("(?Pfirst)|(?Psecond)", + "first") + .expand(r"\2 \g"), + " ") + + def test_repeat_minmax(self): + self.assertIsNone(re.match(r"^(\w){1}$", "abc")) + self.assertIsNone(re.match(r"^(\w){1}?$", "abc")) + self.assertIsNone(re.match(r"^(\w){1,2}$", "abc")) + self.assertIsNone(re.match(r"^(\w){1,2}?$", "abc")) + + self.assertEqual(re.match(r"^(\w){3}$", "abc").group(1), "c") + self.assertEqual(re.match(r"^(\w){1,3}$", "abc").group(1), "c") + self.assertEqual(re.match(r"^(\w){1,4}$", "abc").group(1), "c") + self.assertEqual(re.match(r"^(\w){3,4}?$", "abc").group(1), "c") + self.assertEqual(re.match(r"^(\w){3}?$", "abc").group(1), "c") + self.assertEqual(re.match(r"^(\w){1,3}?$", "abc").group(1), "c") + self.assertEqual(re.match(r"^(\w){1,4}?$", "abc").group(1), "c") + self.assertEqual(re.match(r"^(\w){3,4}?$", "abc").group(1), "c") + + self.assertIsNone(re.match(r"^x{1}$", "xxx")) + self.assertIsNone(re.match(r"^x{1}?$", "xxx")) + self.assertIsNone(re.match(r"^x{1,2}$", "xxx")) + self.assertIsNone(re.match(r"^x{1,2}?$", "xxx")) + + self.assertTrue(re.match(r"^x{3}$", "xxx")) + self.assertTrue(re.match(r"^x{1,3}$", "xxx")) + self.assertTrue(re.match(r"^x{3,3}$", "xxx")) + self.assertTrue(re.match(r"^x{1,4}$", "xxx")) + self.assertTrue(re.match(r"^x{3,4}?$", "xxx")) + self.assertTrue(re.match(r"^x{3}?$", "xxx")) + self.assertTrue(re.match(r"^x{1,3}?$", "xxx")) + self.assertTrue(re.match(r"^x{1,4}?$", "xxx")) + self.assertTrue(re.match(r"^x{3,4}?$", "xxx")) + + # javascript unicode mode won't compile these using \{ would work so no problem not supporting this + # self.assertIsNone(re.match(r"^x{}$", "xxx")) + # self.assertTrue(re.match(r"^x{}$", "x{}")) + + self.checkPatternError(r'x{2,1}', + 'min repeat greater than max repeat', 2) + + def test_getattr(self): + self.assertEqual(re.compile("(?i)(a)(b)").pattern, "(?i)(a)(b)") + self.assertEqual(re.compile("(?i)(a)(b)").flags, re.I | re.U) + # @TODO + self.assertEqual(re.compile("(?i)(a)(b)").groups, 2) + self.assertEqual(re.compile("(?i)(a)(b)").groupindex, {}) + self.assertEqual(re.compile("(?i)(?Pa)(?Pb)").groupindex, + {'first': 1, 'other': 2}) + + self.assertEqual(re.match("(a)", "a").pos, 0) + self.assertEqual(re.match("(a)", "a").endpos, 1) + self.assertEqual(re.match("(a)", "a").string, "a") + self.assertEqual(re.match("(a)", "a").regs, ((0, 1), (0, 1))) + self.assertTrue(re.match("(a)", "a").re) + + # Issue 14260. groupindex should be non-modifiable mapping. + p = re.compile(r'(?i)(?Pa)(?Pb)') + self.assertEqual(sorted(p.groupindex), ['first', 'other']) + self.assertEqual(p.groupindex['other'], 2) + with self.assertRaises(TypeError): + p.groupindex['other'] = 0 + self.assertEqual(p.groupindex['other'], 2) + + def test_special_escapes(self): + self.assertEqual(re.search(r"\b(b.)\b", + "abcd abc bcd bx").group(1), "bx") + self.assertEqual(re.search(r"\B(b.)\B", + "abc bcd bc abxd").group(1), "bx") + self.assertEqual(re.search(r"\b(b.)\b", + "abcd abc bcd bx", re.ASCII).group(1), "bx") + self.assertEqual(re.search(r"\B(b.)\B", + "abc bcd bc abxd", re.ASCII).group(1), "bx") + self.assertEqual(re.search(r"^abc$", "\nabc\n", re.M).group(0), "abc") + self.assertEqual(re.search(r"^\Aabc\Z$", "abc", re.M).group(0), "abc") + self.assertIsNone(re.search(r"^\Aabc\Z$", "\nabc\n", re.M)) + # self.assertEqual(re.search(br"\b(b.)\b", + # b"abcd abc bcd bx").group(1), b"bx") + # self.assertEqual(re.search(br"\B(b.)\B", + # b"abc bcd bc abxd").group(1), b"bx") + # self.assertEqual(re.search(br"\b(b.)\b", + # b"abcd abc bcd bx", re.LOCALE).group(1), b"bx") + # self.assertEqual(re.search(br"\B(b.)\B", + # b"abc bcd bc abxd", re.LOCALE).group(1), b"bx") + # self.assertEqual(re.search(br"^abc$", b"\nabc\n", re.M).group(0), b"abc") + # self.assertEqual(re.search(br"^\Aabc\Z$", b"abc", re.M).group(0), b"abc") + # self.assertIsNone(re.search(br"^\Aabc\Z$", b"\nabc\n", re.M)) + self.assertEqual(re.search(r"\d\D\w\W\s\S", + "1aa! a").group(0), "1aa! a") + # self.assertEqual(re.search(br"\d\D\w\W\s\S", + # b"1aa! a").group(0), b"1aa! a") + # self.assertEqual(re.search(r"\d\D\w\W\s\S", + # "1aa! a", re.ASCII).group(0), "1aa! a") + # self.assertEqual(re.search(br"\d\D\w\W\s\S", + # b"1aa! a", re.LOCALE).group(0), b"1aa! a") + + def test_other_escapes(self): + self.checkPatternError("\\", 'bad escape (end of pattern)', 0) + self.assertEqual(re.match(r"\(", '(').group(), '(') + self.assertIsNone(re.match(r"\(", ')')) + self.assertEqual(re.match(r"\\", '\\').group(), '\\') + self.assertEqual(re.match(r"[\]]", ']').group(), ']') + self.assertIsNone(re.match(r"[\]]", '[')) + self.assertEqual(re.match(r"[a\-c]", '-').group(), '-') + self.assertIsNone(re.match(r"[a\-c]", 'b')) + self.assertEqual(re.match(r"[\^a]+", 'a^').group(), 'a^') + self.assertIsNone(re.match(r"[\^a]+", 'b')) + re.purge() # for warnings + for c in 'ceghijklmopqyzCEFGHIJKLMNOPQRTVXY': + # with self.subTest(c): + self.assertRaises(re.error, re.compile, '\\%c' % c) + for c in 'ceghijklmopqyzABCEFGHIJKLMNOPQRTVXYZ': + # with self.subTest(c): + self.assertRaises(re.error, re.compile, '[\\%c]' % c) + + # def test_named_unicode_escapes(self): + # # test individual Unicode named escapes + # self.assertTrue(re.match(r'\N{LESS-THAN SIGN}', '<')) + # self.assertTrue(re.match(r'\N{less-than sign}', '<')) + # self.assertIsNone(re.match(r'\N{LESS-THAN SIGN}', '>')) + # self.assertTrue(re.match(r'\N{SNAKE}', '\U0001f40d')) + # self.assertTrue(re.match(r'\N{ARABIC LIGATURE UIGHUR KIRGHIZ YEH WITH ' + # r'HAMZA ABOVE WITH ALEF MAKSURA ISOLATED FORM}', + # '\ufbf9')) + # self.assertTrue(re.match(r'[\N{LESS-THAN SIGN}-\N{GREATER-THAN SIGN}]', + # '=')) + # self.assertIsNone(re.match(r'[\N{LESS-THAN SIGN}-\N{GREATER-THAN SIGN}]', + # ';')) + + # # test errors in \N{name} handling - only valid names should pass + # self.checkPatternError(r'\N', 'missing {', 2) + # self.checkPatternError(r'[\N]', 'missing {', 3) + # self.checkPatternError(r'\N{', 'missing character name', 3) + # self.checkPatternError(r'[\N{', 'missing character name', 4) + # self.checkPatternError(r'\N{}', 'missing character name', 3) + # self.checkPatternError(r'[\N{}]', 'missing character name', 4) + # self.checkPatternError(r'\NSNAKE}', 'missing {', 2) + # self.checkPatternError(r'[\NSNAKE}]', 'missing {', 3) + # self.checkPatternError(r'\N{SNAKE', + # 'missing }, unterminated name', 3) + # self.checkPatternError(r'[\N{SNAKE]', + # 'missing }, unterminated name', 4) + # self.checkPatternError(r'[\N{SNAKE]}', + # "undefined character name 'SNAKE]'", 1) + # self.checkPatternError(r'\N{SPAM}', + # "undefined character name 'SPAM'", 0) + # self.checkPatternError(r'[\N{SPAM}]', + # "undefined character name 'SPAM'", 1) + # self.checkPatternError(br'\N{LESS-THAN SIGN}', r'bad escape \N', 0) + # self.checkPatternError(br'[\N{LESS-THAN SIGN}]', r'bad escape \N', 1) + + def test_string_boundaries(self): + # See http://bugs.python.org/issue10713 + self.assertEqual(re.search(r"\b(abc)\b", "abc").group(1), + "abc") + # There's a word boundary at the start of a string. + self.assertTrue(re.match(r"\b", "abc")) + # A non-empty string includes a non-boundary zero-length match. + self.assertTrue(re.search(r"\B", "abc")) + # There is no non-boundary match at the start of a string. + self.assertFalse(re.match(r"\B", "abc")) + # However, an empty string contains no word boundaries, and also no + # non-boundaries. + # self.assertIsNone(re.search(r"\B", "")) + # This one is questionable and different from the perlre behaviour, + # but describes current behavior. + self.assertIsNone(re.search(r"\b", "")) + # A single word-character string has two boundaries, but no + # non-boundary gaps. + self.assertEqual(len(re.findall(r"\b", "a")), 2) + self.assertEqual(len(re.findall(r"\B", "a")), 0) + # If there are no words, there are no boundaries + self.assertEqual(len(re.findall(r"\b", " ")), 0) + self.assertEqual(len(re.findall(r"\b", " ")), 0) + # Can match around the whitespace. + self.assertEqual(len(re.findall(r"\B", " ")), 2) + + def test_bigcharset(self): + self.assertEqual(re.match("([\u2222\u2223])", + "\u2222").group(1), "\u2222") + r = '[%s]' % ''.join(map(chr, range(256, 2**16, 255))) + self.assertEqual(re.match(r, "\uff01").group(), "\uff01") + + def test_big_codesize(self): + # Issue #1160 + r = re.compile('|'.join(('%d'%x for x in range(10000)))) + self.assertTrue(r.match('1000')) + self.assertTrue(r.match('9999')) + + def test_anyall(self): + self.assertEqual(re.match("a.b", "a\nb", re.DOTALL).group(0), + "a\nb") + self.assertEqual(re.match("a.*b", "a\n\nb", re.DOTALL).group(0), + "a\n\nb") + + def test_lookahead(self): + self.assertEqual(re.match(r"(a(?=\s[^a]))", "a b").group(1), "a") + self.assertEqual(re.match(r"(a(?=\s[^a]*))", "a b").group(1), "a") + self.assertEqual(re.match(r"(a(?=\s[abc]))", "a b").group(1), "a") + self.assertEqual(re.match(r"(a(?=\s[abc]*))", "a bc").group(1), "a") + self.assertEqual(re.match(r"(a)(?=\s\1)", "a a").group(1), "a") + self.assertEqual(re.match(r"(a)(?=\s\1*)", "a aa").group(1), "a") + self.assertEqual(re.match(r"(a)(?=\s(abc|a))", "a a").group(1), "a") + + self.assertEqual(re.match(r"(a(?!\s[^a]))", "a a").group(1), "a") + self.assertEqual(re.match(r"(a(?!\s[abc]))", "a d").group(1), "a") + self.assertEqual(re.match(r"(a)(?!\s\1)", "a b").group(1), "a") + self.assertEqual(re.match(r"(a)(?!\s(abc|a))", "a b").group(1), "a") + + # Group reference. + self.assertTrue(re.match(r'(a)b(?=\1)a', 'aba')) + self.assertIsNone(re.match(r'(a)b(?=\1)c', 'abac')) + # @TODO + # # Conditional group reference. + # self.assertTrue(re.match(r'(?:(a)|(x))b(?=(?(2)x|c))c', 'abc')) + # self.assertIsNone(re.match(r'(?:(a)|(x))b(?=(?(2)c|x))c', 'abc')) + # self.assertTrue(re.match(r'(?:(a)|(x))b(?=(?(2)x|c))c', 'abc')) + # self.assertIsNone(re.match(r'(?:(a)|(x))b(?=(?(1)b|x))c', 'abc')) + # self.assertTrue(re.match(r'(?:(a)|(x))b(?=(?(1)c|x))c', 'abc')) + # # Group used before defined. + # self.assertTrue(re.match(r'(a)b(?=(?(2)x|c))(c)', 'abc')) + # self.assertIsNone(re.match(r'(a)b(?=(?(2)b|x))(c)', 'abc')) + # self.assertTrue(re.match(r'(a)b(?=(?(1)c|x))(c)', 'abc')) + + def test_lookbehind(self): + self.assertTrue(re.match(r'ab(?<=b)c', 'abc')) + self.assertIsNone(re.match(r'ab(?<=c)c', 'abc')) + self.assertIsNone(re.match(r'ab(?.)(?P=a))(c)') + # self.assertRaises(re.error, re.compile, r'(a)b(?<=(a)(?(2)b|x))(c)') + # self.assertRaises(re.error, re.compile, r'(a)b(?<=(.)(?<=\2))(c)') + + def test_ignore_case(self): + self.assertEqual(re.match("abc", "ABC", re.I).group(0), "ABC") + # self.assertEqual(re.match(b"abc", b"ABC", re.I).group(0), b"ABC") + self.assertEqual(re.match(r"(a\s[^a])", "a b", re.I).group(1), "a b") + self.assertEqual(re.match(r"(a\s[^a]*)", "a bb", re.I).group(1), "a bb") + self.assertEqual(re.match(r"(a\s[abc])", "a b", re.I).group(1), "a b") + self.assertEqual(re.match(r"(a\s[abc]*)", "a bb", re.I).group(1), "a bb") + self.assertEqual(re.match(r"((a)\s\2)", "a a", re.I).group(1), "a a") + self.assertEqual(re.match(r"((a)\s\2*)", "a aa", re.I).group(1), "a aa") + self.assertEqual(re.match(r"((a)\s(abc|a))", "a a", re.I).group(1), "a a") + self.assertEqual(re.match(r"((a)\s(abc|a)*)", "a aa", re.I).group(1), "a aa") + + assert '\u212a'.lower() == 'k' # 'K' + self.assertTrue(re.match(r'K', '\u212a', re.I)) + self.assertTrue(re.match(r'k', '\u212a', re.I)) + self.assertTrue(re.match(r'\u212a', 'K', re.I)) + self.assertTrue(re.match(r'\u212a', 'k', re.I)) + assert '\u017f'.upper() == 'S' # 'ſ' + self.assertTrue(re.match(r'S', '\u017f', re.I)) + self.assertTrue(re.match(r's', '\u017f', re.I)) + self.assertTrue(re.match(r'\u017f', 'S', re.I)) + self.assertTrue(re.match(r'\u017f', 's', re.I)) + assert '\ufb05'.upper() == '\ufb06'.upper() == 'ST' # 'ſt', 'st' + self.assertTrue(re.match(r'\ufb05', '\ufb06', re.I)) + self.assertTrue(re.match(r'\ufb06', '\ufb05', re.I)) + + def test_ignore_case_set(self): + self.assertTrue(re.match(r'[19A]', 'A', re.I)) + self.assertTrue(re.match(r'[19a]', 'a', re.I)) + self.assertTrue(re.match(r'[19a]', 'A', re.I)) + self.assertTrue(re.match(r'[19A]', 'a', re.I)) + # self.assertTrue(re.match(br'[19A]', b'A', re.I)) + # self.assertTrue(re.match(br'[19a]', b'a', re.I)) + # self.assertTrue(re.match(br'[19a]', b'A', re.I)) + # self.assertTrue(re.match(br'[19A]', b'a', re.I)) + assert '\u212a'.lower() == 'k' # 'K' + self.assertTrue(re.match(r'[19K]', '\u212a', re.I)) + self.assertTrue(re.match(r'[19k]', '\u212a', re.I)) + self.assertTrue(re.match(r'[19\u212a]', 'K', re.I)) + self.assertTrue(re.match(r'[19\u212a]', 'k', re.I)) + assert '\u017f'.upper() == 'S' # 'ſ' + self.assertTrue(re.match(r'[19S]', '\u017f', re.I)) + self.assertTrue(re.match(r'[19s]', '\u017f', re.I)) + self.assertTrue(re.match(r'[19\u017f]', 'S', re.I)) + self.assertTrue(re.match(r'[19\u017f]', 's', re.I)) + assert '\ufb05'.upper() == '\ufb06'.upper() == 'ST' # 'ſt', 'st' + self.assertTrue(re.match(r'[19\ufb05]', '\ufb06', re.I)) + self.assertTrue(re.match(r'[19\ufb06]', '\ufb05', re.I)) + + def test_ignore_case_range(self): + # Issues #3511, #17381. + self.assertTrue(re.match(r'[9-a]', '_', re.I)) + # self.assertIsNone(re.match(r'[9-A]', '_', re.I)) # throws an error + # self.assertTrue(re.match(br'[9-a]', b'_', re.I)) + # self.assertIsNone(re.match(br'[9-A]', b'_', re.I)) + self.assertTrue(re.match(r'[\xc0-\xde]', '\xd7', re.I)) + self.assertIsNone(re.match(r'[\xc0-\xde]', '\xf7', re.I)) + self.assertTrue(re.match(r'[\xe0-\xfe]', '\xf7', re.I)) + self.assertIsNone(re.match(r'[\xe0-\xfe]', '\xd7', re.I)) + self.assertTrue(re.match(r'[\u0430-\u045f]', '\u0450', re.I)) + self.assertTrue(re.match(r'[\u0430-\u045f]', '\u0400', re.I)) + self.assertTrue(re.match(r'[\u0400-\u042f]', '\u0450', re.I)) + self.assertTrue(re.match(r'[\u0400-\u042f]', '\u0400', re.I)) + # these 4 tests should have r' but does not compile with r' + self.assertTrue(re.match('[\U00010428-\U0001044f]', '\U00010428', re.I)) + self.assertTrue(re.match('[\U00010428-\U0001044f]', '\U00010400', re.I)) + self.assertTrue(re.match('[\U00010400-\U00010427]', '\U00010428', re.I)) + self.assertTrue(re.match('[\U00010400-\U00010427]', '\U00010400', re.I)) + # self.assertTrue(re.match(r'[\U00010428-\U0001044f]', '\U00010428', re.I)) + # self.assertTrue(re.match(r'[\U00010428-\U0001044f]', '\U00010400', re.I)) + # self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010428', re.I)) + # self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010400', re.I)) + + assert '\u212a'.lower() == 'k' # 'K' + self.assertTrue(re.match(r'[J-M]', '\u212a', re.I)) + self.assertTrue(re.match(r'[j-m]', '\u212a', re.I)) + self.assertTrue(re.match(r'[\u2129-\u212b]', 'K', re.I)) + self.assertTrue(re.match(r'[\u2129-\u212b]', 'k', re.I)) + assert '\u017f'.upper() == 'S' # 'ſ' + self.assertTrue(re.match(r'[R-T]', '\u017f', re.I)) + self.assertTrue(re.match(r'[r-t]', '\u017f', re.I)) + self.assertTrue(re.match(r'[\u017e-\u0180]', 'S', re.I)) + self.assertTrue(re.match(r'[\u017e-\u0180]', 's', re.I)) + assert '\ufb05'.upper() == '\ufb06'.upper() == 'ST' # 'ſt', 'st' + self.assertTrue(re.match(r'[\ufb04-\ufb05]', '\ufb06', re.I)) + self.assertTrue(re.match(r'[\ufb06-\ufb07]', '\ufb05', re.I)) + + def test_category(self): + self.assertEqual(re.match(r"(\s)", " ").group(1), " ") + + # @cpython_only + # def test_case_helpers(self): + # import _sre + # for i in range(128): + # c = chr(i) + # lo = ord(c.lower()) + # self.assertEqual(_sre.ascii_tolower(i), lo) + # self.assertEqual(_sre.unicode_tolower(i), lo) + # iscased = c in string.ascii_letters + # self.assertEqual(_sre.ascii_iscased(i), iscased) + # self.assertEqual(_sre.unicode_iscased(i), iscased) + + # for i in list(range(128, 0x1000)) + [0x10400, 0x10428]: + # c = chr(i) + # self.assertEqual(_sre.ascii_tolower(i), i) + # if i != 0x0130: + # self.assertEqual(_sre.unicode_tolower(i), ord(c.lower())) + # iscased = c != c.lower() or c != c.upper() + # self.assertFalse(_sre.ascii_iscased(i)) + # self.assertEqual(_sre.unicode_iscased(i), + # c != c.lower() or c != c.upper()) + + # self.assertEqual(_sre.ascii_tolower(0x0130), 0x0130) + # self.assertEqual(_sre.unicode_tolower(0x0130), ord('i')) + # self.assertFalse(_sre.ascii_iscased(0x0130)) + # self.assertTrue(_sre.unicode_iscased(0x0130)) + + def test_not_literal(self): + self.assertEqual(re.search(r"\s([^a])", " b").group(1), "b") + self.assertEqual(re.search(r"\s([^a]*)", " bb").group(1), "bb") + + def test_possible_set_operations(self): + s = bytes(range(128)).decode() + # with self.assertWarns(FutureWarning): + p = re.compile(r'[0-9--1]') + self.assertEqual(p.findall(s), list('-./0123456789')) + self.assertEqual(re.findall(r'[--1]', s), list('-./01')) + # with self.assertWarns(FutureWarning): + p = re.compile(r'[%--1]') + self.assertEqual(p.findall(s), list("%&'()*+,-1")) + # with self.assertWarns(FutureWarning): + p = re.compile(r'[%--]') + self.assertEqual(p.findall(s), list("%&'()*+,-")) + + # with self.assertWarns(FutureWarning): + p = re.compile(r'[0-9&&1]') + self.assertEqual(p.findall(s), list('&0123456789')) + # with self.assertWarns(FutureWarning): + p = re.compile(r'[\d&&1]') + self.assertEqual(p.findall(s), list('&0123456789')) + self.assertEqual(re.findall(r'[&&1]', s), list('&1')) + + # with self.assertWarns(FutureWarning): + p = re.compile(r'[0-9||a]') + self.assertEqual(p.findall(s), list('0123456789a|')) + # with self.assertWarns(FutureWarning): + p = re.compile(r'[\d||a]') + self.assertEqual(p.findall(s), list('0123456789a|')) + self.assertEqual(re.findall(r'[||1]', s), list('1|')) + + # with self.assertWarns(FutureWarning): + p = re.compile(r'[0-9~~1]') + self.assertEqual(p.findall(s), list('0123456789~')) + # with self.assertWarns(FutureWarning): + p = re.compile(r'[\d~~1]') + self.assertEqual(p.findall(s), list('0123456789~')) + self.assertEqual(re.findall(r'[~~1]', s), list('1~')) + + # # with self.assertWarns(FutureWarning): + # p = re.compile(r'[[0-9]|]') + # self.assertEqual(p.findall(s), list('0123456789[]')) + + # # with self.assertWarns(FutureWarning): + # p = re.compile(r'[[:digit:]|]') + # self.assertEqual(p.findall(s), list(':[]dgit')) + + def test_search_coverage(self): + self.assertEqual(re.search(r"\s(b)", " b").group(1), "b") + self.assertEqual(re.search(r"a\s", "a ").group(0), "a ") + + def assertMatch(self, pattern, text, match=None, span=None, + matcher=re.fullmatch): + if match is None and span is None: + # the pattern matches the whole text + match = text + span = (0, len(text)) + elif match is None or span is None: + raise ValueError('If match is not None, span should be specified ' + '(and vice versa).') + m = matcher(pattern, text) + self.assertTrue(m) + self.assertEqual(m.group(), match) + self.assertEqual(m.span(), span) + + LITERAL_CHARS = string.ascii_letters + string.digits + '!"%\',/:;<=>@_`' + + def test_re_escape(self): + p = ''.join(chr(i) for i in range(256)) + for c in p: + self.assertMatch(re.escape(c), c) + self.assertMatch('[' + re.escape(c) + ']', c) + self.assertMatch('(?x)' + re.escape(c), c) + self.assertMatch(re.escape(p), p) + for c in '-.]{}': + self.assertEqual(re.escape(c)[:1], '\\') + literal_chars = self.LITERAL_CHARS + self.assertEqual(re.escape(literal_chars), literal_chars) + + # def test_re_escape_bytes(self): + # p = bytes(range(256)) + # for i in p: + # b = bytes([i]) + # self.assertMatch(re.escape(b), b) + # self.assertMatch(b'[' + re.escape(b) + b']', b) + # self.assertMatch(b'(?x)' + re.escape(b), b) + # self.assertMatch(re.escape(p), p) + # for i in b'-.]{}': + # b = bytes([i]) + # self.assertEqual(re.escape(b)[:1], b'\\') + # literal_chars = self.LITERAL_CHARS.encode('ascii') + # self.assertEqual(re.escape(literal_chars), literal_chars) + + def test_re_escape_non_ascii(self): + s = 'xxx\u2620\u2620\u2620xxx' + s_escaped = re.escape(s) + self.assertEqual(s_escaped, s) + self.assertMatch(s_escaped, s) + self.assertMatch('.%s+.' % re.escape('\u2620'), s, + 'x\u2620\u2620\u2620x', (2, 7), re.search) + + # def test_re_escape_non_ascii_bytes(self): + # b = 'y\u2620y\u2620y'.encode('utf-8') + # b_escaped = re.escape(b) + # self.assertEqual(b_escaped, b) + # self.assertMatch(b_escaped, b) + # res = re.findall(re.escape('\u2620'.encode('utf-8')), b) + # self.assertEqual(len(res), 2) + + # def test_pickling(self): + # import pickle + # oldpat = re.compile('a(?:b|(c|e){1,2}?|d)+?(.)', re.UNICODE) + # for proto in range(pickle.HIGHEST_PROTOCOL + 1): + # pickled = pickle.dumps(oldpat, proto) + # newpat = pickle.loads(pickled) + # self.assertEqual(newpat, oldpat) + # # current pickle expects the _compile() reconstructor in re module + # from re import _compile + + def test_copying(self): + import copy + p = re.compile(r'(?P\d+)(?:\.(?P\d*))?') + self.assertIs(copy.copy(p), p) + self.assertIs(copy.deepcopy(p), p) + m = p.match('12.34') + self.assertIs(copy.copy(m), m) + self.assertIs(copy.deepcopy(m), m) + + def test_constants(self): + self.assertEqual(re.I, re.IGNORECASE) + self.assertEqual(re.L, re.LOCALE) + self.assertEqual(re.M, re.MULTILINE) + self.assertEqual(re.S, re.DOTALL) + self.assertEqual(re.X, re.VERBOSE) + + def test_flags(self): + for flag in [re.I, re.M, re.X, re.S, re.A, re.U]: + self.assertTrue(re.compile('^pattern$', flag)) + # for flag in [re.I, re.M, re.X, re.S, re.A, re.L]: + # self.assertTrue(re.compile(b'^pattern$', flag)) + + def test_sre_character_literals(self): + for i in [0, 8, 16, 32, 64, 127, 128, 255, 256, 0xFFFF, 0x10000, 0x10FFFF]: + if i < 256: + pass + # self.assertTrue(re.match(r"\%03o" % i, chr(i))) + # self.assertTrue(re.match(r"\%03o0" % i, chr(i)+"0")) + # self.assertTrue(re.match(r"\%03o8" % i, chr(i)+"8")) + # self.assertTrue(re.match(r"\x%02x" % i, chr(i))) + # self.assertTrue(re.match(r"\x%02x0" % i, chr(i)+"0")) + # self.assertTrue(re.match(r"\x%02xz" % i, chr(i)+"z")) + if i < 0x10000: + self.assertTrue(re.match(r"\u%04x" % i, chr(i))) + self.assertTrue(re.match(r"\u%04x0" % i, chr(i)+"0")) + self.assertTrue(re.match(r"\u%04xz" % i, chr(i)+"z")) + # self.assertTrue(re.match(r"\U%08x" % i, chr(i))) + # self.assertTrue(re.match(r"\U%08x0" % i, chr(i)+"0")) + # self.assertTrue(re.match(r"\U%08xz" % i, chr(i)+"z")) + self.assertTrue(re.match(r"\0", "\000")) + # self.assertTrue(re.match(r"\08", "\0008")) + # self.assertTrue(re.match(r"\01", "\001")) + # self.assertTrue(re.match(r"\018", "\0018")) + self.checkPatternError(r"\567", + r'octal escape value \567 outside of ' + r'range 0-0o377', 0) + self.checkPatternError(r"\911", 'invalid group reference 91', 1) + self.checkPatternError(r"\x1", r'incomplete escape \x1', 0) + self.checkPatternError(r"\x1z", r'incomplete escape \x1', 0) + self.checkPatternError(r"\u123", r'incomplete escape \u123', 0) + self.checkPatternError(r"\u123z", r'incomplete escape \u123', 0) + self.checkPatternError(r"\U0001234", r'incomplete escape \U0001234', 0) + self.checkPatternError(r"\U0001234z", r'incomplete escape \U0001234', 0) + self.checkPatternError(r"\U00110000", r'bad escape \U00110000', 0) + + def test_sre_character_class_literals(self): + for i in [0, 8, 16, 32, 64, 127, 128, 255, 256, 0xFFFF, 0x10000, 0x10FFFF]: + if i < 256: + pass + # self.assertTrue(re.match(r"[\%o]" % i, chr(i))) + # self.assertTrue(re.match(r"[\%o8]" % i, chr(i))) + # self.assertTrue(re.match(r"[\%03o]" % i, chr(i))) + # self.assertTrue(re.match(r"[\%03o0]" % i, chr(i))) + # self.assertTrue(re.match(r"[\%03o8]" % i, chr(i))) + # self.assertTrue(re.match(r"[\x%02x]" % i, chr(i))) + # self.assertTrue(re.match(r"[\x%02x0]" % i, chr(i))) + # self.assertTrue(re.match(r"[\x%02xz]" % i, chr(i))) + if i < 0x10000: + self.assertTrue(re.match(r"[\u%04x]" % i, chr(i))) + self.assertTrue(re.match(r"[\u%04x0]" % i, chr(i))) + self.assertTrue(re.match(r"[\u%04xz]" % i, chr(i))) + # self.assertTrue(re.match(r"[\U%08x]" % i, chr(i))) + # self.assertTrue(re.match(r"[\U%08x0]" % i, chr(i)+"0")) + # self.assertTrue(re.match(r"[\U%08xz]" % i, chr(i)+"z")) + self.checkPatternError(r"[\567]", + r'octal escape value \567 outside of ' + r'range 0-0o377', 1) + self.checkPatternError(r"[\911]", r'bad escape \9', 1) + self.checkPatternError(r"[\x1z]", r'incomplete escape \x1', 1) + self.checkPatternError(r"[\u123z]", r'incomplete escape \u123', 1) + self.checkPatternError(r"[\U0001234z]", r'incomplete escape \U0001234', 1) + self.checkPatternError(r"[\U00110000]", r'bad escape \U00110000', 1) + # this doesn't get compiled correctly in raw mode + # self.assertTrue(re.match(r"[\U0001d49c-\U0001d4b5]", "\U0001d49e")) + self.assertTrue(re.match("[\U0001d49c-\U0001d4b5]", "\U0001d49e")) + + # def test_sre_byte_literals(self): + # for i in [0, 8, 16, 32, 64, 127, 128, 255]: + # self.assertTrue(re.match((r"\%03o" % i).encode(), bytes([i]))) + # self.assertTrue(re.match((r"\%03o0" % i).encode(), bytes([i])+b"0")) + # self.assertTrue(re.match((r"\%03o8" % i).encode(), bytes([i])+b"8")) + # self.assertTrue(re.match((r"\x%02x" % i).encode(), bytes([i]))) + # self.assertTrue(re.match((r"\x%02x0" % i).encode(), bytes([i])+b"0")) + # self.assertTrue(re.match((r"\x%02xz" % i).encode(), bytes([i])+b"z")) + # self.assertRaises(re.error, re.compile, br"\u1234") + # self.assertRaises(re.error, re.compile, br"\U00012345") + # self.assertTrue(re.match(br"\0", b"\000")) + # self.assertTrue(re.match(br"\08", b"\0008")) + # self.assertTrue(re.match(br"\01", b"\001")) + # self.assertTrue(re.match(br"\018", b"\0018")) + # self.checkPatternError(br"\567", + # r'octal escape value \567 outside of ' + # r'range 0-0o377', 0) + # self.checkPatternError(br"\911", 'invalid group reference 91', 1) + # self.checkPatternError(br"\x1", r'incomplete escape \x1', 0) + # self.checkPatternError(br"\x1z", r'incomplete escape \x1', 0) + + # def test_sre_byte_class_literals(self): + # for i in [0, 8, 16, 32, 64, 127, 128, 255]: + # self.assertTrue(re.match((r"[\%o]" % i).encode(), bytes([i]))) + # self.assertTrue(re.match((r"[\%o8]" % i).encode(), bytes([i]))) + # self.assertTrue(re.match((r"[\%03o]" % i).encode(), bytes([i]))) + # self.assertTrue(re.match((r"[\%03o0]" % i).encode(), bytes([i]))) + # self.assertTrue(re.match((r"[\%03o8]" % i).encode(), bytes([i]))) + # self.assertTrue(re.match((r"[\x%02x]" % i).encode(), bytes([i]))) + # self.assertTrue(re.match((r"[\x%02x0]" % i).encode(), bytes([i]))) + # self.assertTrue(re.match((r"[\x%02xz]" % i).encode(), bytes([i]))) + # self.assertRaises(re.error, re.compile, br"[\u1234]") + # self.assertRaises(re.error, re.compile, br"[\U00012345]") + # self.checkPatternError(br"[\567]", + # r'octal escape value \567 outside of ' + # r'range 0-0o377', 1) + # self.checkPatternError(br"[\911]", r'bad escape \9', 1) + # self.checkPatternError(br"[\x1z]", r'incomplete escape \x1', 1) + + def test_character_set_errors(self): + self.checkPatternError(r'[', 'unterminated character set', 0) + self.checkPatternError(r'[^', 'unterminated character set', 0) + self.checkPatternError(r'[a', 'unterminated character set', 0) + # bug 545855 -- This pattern failed to cause a compile error as it + # should, instead provoking a TypeError. + self.checkPatternError(r"[a-", 'unterminated character set', 0) + self.checkPatternError(r"[\w-b]", r'bad character range \w-b', 1) + self.checkPatternError(r"[a-\w]", r'bad character range a-\w', 1) + self.checkPatternError(r"[b-a]", 'bad character range b-a', 1) + + def test_bug_113254(self): + self.assertEqual(re.match(r'(a)|(b)', 'b').start(1), -1) + self.assertEqual(re.match(r'(a)|(b)', 'b').end(1), -1) + self.assertEqual(re.match(r'(a)|(b)', 'b').span(1), (-1, -1)) + + def test_bug_527371(self): + # bug described in patches 527371/672491 + self.assertIsNone(re.match(r'(a)?a','a').lastindex) + self.assertEqual(re.match(r'(a)(b)?b','ab').lastindex, 1) + self.assertEqual(re.match(r'(?Pa)(?Pb)?b','ab').lastgroup, 'a') + self.assertEqual(re.match(r"(?Pa(b))", "ab").lastgroup, 'a') + self.assertEqual(re.match(r"((a))", "a").lastindex, 1) + + def test_bug_418626(self): + # bugs 418626 at al. -- Testing Greg Chapman's addition of op code + # SRE_OP_MIN_REPEAT_ONE for eliminating recursion on simple uses of + # pattern '*?' on a long string. + self.assertEqual(re.match('.*?c', 10000*'ab'+'cd').end(0), 20001) + self.assertEqual(re.match('.*?cd', 5000*'ab'+'c'+5000*'ab'+'cde').end(0), + 20003) + self.assertEqual(re.match('.*?cd', 20000*'abc'+'de').end(0), 60001) + # non-simple '*?' still used to hit the recursion limit, before the + # non-recursive scheme was implemented. + self.assertEqual(re.search('(a|b)*?c', 10000*'ab'+'cd').end(0), 20001) + + def test_bug_612074(self): + pat="["+re.escape("\u2039")+"]" + self.assertEqual(re.compile(pat) and 1, 1) + + def test_stack_overflow(self): + # nasty cases that used to overflow the straightforward recursive + # implementation of repeated groups. + self.assertEqual(re.match('(x)*', 50000*'x').group(1), 'x') + self.assertEqual(re.match('(x)*y', 50000*'x'+'y').group(1), 'x') + self.assertEqual(re.match('(x)*?y', 50000*'x'+'y').group(1), 'x') + + def test_nothing_to_repeat(self): + for reps in '*', '+', '?', '{1,2}': + for mod in '', '?': + self.checkPatternError('%s%s' % (reps, mod), + 'nothing to repeat', 0) + self.checkPatternError('(?:%s%s)' % (reps, mod), + 'nothing to repeat', 3) + + def test_multiple_repeat(self): + for outer_reps in '*', '+', '{1,2}': + for outer_mod in '', '?': + outer_op = outer_reps + outer_mod + for inner_reps in '*', '+', '?', '{1,2}': + for inner_mod in '', '?': + inner_op = inner_reps + inner_mod + self.checkPatternError(r'x%s%s' % (inner_op, outer_op), + 'multiple repeat', 1 + len(inner_op)) + + def test_unlimited_zero_width_repeat(self): + # Issue #9669 + self.assertIsNone(re.match(r'(?:a?)*y', 'z')) + self.assertIsNone(re.match(r'(?:a?)+y', 'z')) + self.assertIsNone(re.match(r'(?:a?){2,}y', 'z')) + self.assertIsNone(re.match(r'(?:a?)*?y', 'z')) + self.assertIsNone(re.match(r'(?:a?)+?y', 'z')) + self.assertIsNone(re.match(r'(?:a?){2,}?y', 'z')) + + # def test_scanner(self): + # def s_ident(scanner, token): return token + # def s_operator(scanner, token): return "op%s" % token + # def s_float(scanner, token): return float(token) + # def s_int(scanner, token): return int(token) + + # scanner = Scanner([ + # (r"[a-zA-Z_]\w*", s_ident), + # (r"\d+\.\d*", s_float), + # (r"\d+", s_int), + # (r"=|\+|-|\*|/", s_operator), + # (r"\s+", None), + # ]) + + # self.assertTrue(scanner.scanner.scanner("").pattern) + + # self.assertEqual(scanner.scan("sum = 3*foo + 312.50 + bar"), + # (['sum', 'op=', 3, 'op*', 'foo', 'op+', 312.5, + # 'op+', 'bar'], '')) + + def test_bug_448951(self): + # bug 448951 (similar to 429357, but with single char match) + # (Also test greedy matches.) + for op in '','?','*': + self.assertEqual(re.match(r'((.%s):)?z'%op, 'z').groups(), + (None, None)) + self.assertEqual(re.match(r'((.%s):)?z'%op, 'a:z').groups(), + ('a:', 'a')) + + def test_bug_725106(self): + # capturing groups in alternatives in repeats + # self.assertEqual(re.match('^((a)|b)*', 'abc').groups(), + # ('b', 'a')) + # self.assertEqual(re.match('^(([ab])|c)*', 'abc').groups(), + # ('c', 'b')) + self.assertEqual(re.match('^((d)|[ab])*', 'abc').groups(), + ('b', None)) + self.assertEqual(re.match('^((a)c|[ab])*', 'abc').groups(), + ('b', None)) + # self.assertEqual(re.match('^((a)|b)*?c', 'abc').groups(), + # ('b', 'a')) + # self.assertEqual(re.match('^(([ab])|c)*?d', 'abcd').groups(), + # ('c', 'b')) + self.assertEqual(re.match('^((d)|[ab])*?c', 'abc').groups(), + ('b', None)) + self.assertEqual(re.match('^((a)c|[ab])*?c', 'abc').groups(), + ('b', None)) + + def test_bug_725149(self): + # mark_stack_base restoring before restoring marks + self.assertEqual(re.match('(a)(?:(?=(b)*)c)*', 'abb').groups(), + ('a', None)) + self.assertEqual(re.match('(a)((?!(b)*))*', 'abb').groups(), + ('a', None, None)) + + def test_bug_764548(self): + # bug 764548, re.compile() barfs on str/unicode subclasses + class my_unicode(str): pass + pat = re.compile(my_unicode("abc")) + self.assertIsNone(pat.match("xyz")) + + def test_finditer(self): + iter = re.finditer(r":+", "a:b::c:::d") + self.assertEqual([item.group(0) for item in iter], + [":", "::", ":::"]) + + pat = re.compile(r":+") + iter = pat.finditer("a:b::c:::d", 1, 10) + self.assertEqual([item.group(0) for item in iter], + [":", "::", ":::"]) + + pat = re.compile(r":+") + iter = pat.finditer("a:b::c:::d", pos=1, endpos=10) + self.assertEqual([item.group(0) for item in iter], + [":", "::", ":::"]) + + pat = re.compile(r":+") + iter = pat.finditer("a:b::c:::d", endpos=10, pos=1) + self.assertEqual([item.group(0) for item in iter], + [":", "::", ":::"]) + + pat = re.compile(r":+") + iter = pat.finditer("a:b::c:::d", pos=3, endpos=8) + self.assertEqual([item.group(0) for item in iter], + ["::", "::"]) + + def test_bug_926075(self): + pass + # self.assertIsNot(re.compile('bug_926075'), + # re.compile(b'bug_926075')) + + def test_bug_931848(self): + pattern = "[\u002E\u3002\uFF0E\uFF61]" + self.assertEqual(re.compile(pattern).split("a.b.c"), + ['a','b','c']) + + def test_bug_581080(self): + iter = re.finditer(r"\s", "a b") + self.assertEqual(next(iter).span(), (1,2)) + self.assertRaises(StopIteration, next, iter) + + # scanner = re.compile(r"\s").scanner("a b") + # self.assertEqual(scanner.search().span(), (1, 2)) + # self.assertIsNone(scanner.search()) + + def test_bug_817234(self): + iter = re.finditer(r".*", "asdf") + self.assertEqual(next(iter).span(), (0, 4)) + self.assertEqual(next(iter).span(), (4, 4)) + self.assertRaises(StopIteration, next, iter) + + def test_bug_6561(self): + # '\d' should match characters in Unicode category 'Nd' + # (Number, Decimal Digit), but not those in 'Nl' (Number, + # Letter) or 'No' (Number, Other). + decimal_digits = [ + '\u0037', # '\N{DIGIT SEVEN}', category 'Nd' + # '\u0e58', # '\N{THAI DIGIT SIX}', category 'Nd' + # '\uff10', # '\N{FULLWIDTH DIGIT ZERO}', category 'Nd' + ] + for x in decimal_digits: + self.assertEqual(re.match(r'^\d$', x).group(0), x) + + not_decimal_digits = [ + '\u2165', # '\N{ROMAN NUMERAL SIX}', category 'Nl' + '\u3039', # '\N{HANGZHOU NUMERAL TWENTY}', category 'Nl' + '\u2082', # '\N{SUBSCRIPT TWO}', category 'No' + '\u32b4', # '\N{CIRCLED NUMBER THIRTY NINE}', category 'No' + ] + for x in not_decimal_digits: + self.assertIsNone(re.match(r'^\d$', x)) + + # def test_empty_array(self): + # # SF buf 1647541 + # import array + # for typecode in 'bBuhHiIlLfd': + # a = array.array(typecode) + # self.assertIsNone(re.compile(b"bla").match(a)) + # self.assertEqual(re.compile(b"").match(a).groups(), ()) + + def test_inline_flags(self): + # Bug #1700 + upper_char = '\u1ea0' # Latin Capital Letter A with Dot Below + lower_char = '\u1ea1' # Latin Small Letter A with Dot Below + + p = re.compile('.' + upper_char, re.I | re.S) + q = p.match('\n' + lower_char) + self.assertTrue(q) + + p = re.compile('.' + lower_char, re.I | re.S) + q = p.match('\n' + upper_char) + self.assertTrue(q) + + p = re.compile('(?i).' + upper_char, re.S) + q = p.match('\n' + lower_char) + self.assertTrue(q) + + p = re.compile('(?i).' + lower_char, re.S) + q = p.match('\n' + upper_char) + self.assertTrue(q) + + p = re.compile('(?is).' + upper_char) + q = p.match('\n' + lower_char) + self.assertTrue(q) + + p = re.compile('(?is).' + lower_char) + q = p.match('\n' + upper_char) + self.assertTrue(q) + + p = re.compile('(?s)(?i).' + upper_char) + q = p.match('\n' + lower_char) + self.assertTrue(q) + + p = re.compile('(?s)(?i).' + lower_char) + q = p.match('\n' + upper_char) + self.assertTrue(q) + + # we don't support debug mode + # self.assertTrue(re.match('(?ix) ' + upper_char, lower_char)) + # self.assertTrue(re.match('(?ix) ' + lower_char, upper_char)) + # self.assertTrue(re.match(' (?i) ' + upper_char, lower_char, re.X)) + # self.assertTrue(re.match('(?x) (?i) ' + upper_char, lower_char)) + # self.assertTrue(re.match(' (?x) (?i) ' + upper_char, lower_char, re.X)) + + p = upper_char + '(?i)' + # with self.assertWarns(DeprecationWarning) as warns: + self.assertTrue(re.match(p, lower_char)) + # self.assertEqual( + # str(warns.warnings[0].message), + # 'Flags not at the start of the expression %r' % p + # ) + # self.assertEqual(warns.warnings[0].filename, __file__) + + p = upper_char + '(?i)%s' % ('.?' * 100) + # with self.assertWarns(DeprecationWarning) as warns: + self.assertTrue(re.match(p, lower_char)) + # self.assertEqual( + # str(warns.warnings[0].message), + # 'Flags not at the start of the expression %r (truncated)' % p[:20] + # ) + # self.assertEqual(warns.warnings[0].filename, __file__) + + # # bpo-30605: Compiling a bytes instance regex was throwing a BytesWarning + # with warnings.catch_warnings(): + # warnings.simplefilter('error', BytesWarning) + # p = b'A(?i)' + # with self.assertWarns(DeprecationWarning) as warns: + # self.assertTrue(re.match(p, b'a')) + # self.assertEqual( + # str(warns.warnings[0].message), + # 'Flags not at the start of the expression %r' % p + # ) + # self.assertEqual(warns.warnings[0].filename, __file__) + + # with self.assertWarns(DeprecationWarning): + self.assertTrue(re.match('(?s).(?i)' + upper_char, '\n' + lower_char)) + # with self.assertWarns(DeprecationWarning): + # self.assertTrue(re.match('(?i) ' + upper_char + ' (?x)', lower_char)) + # with self.assertWarns(DeprecationWarning): + # self.assertTrue(re.match(' (?x) (?i) ' + upper_char, lower_char)) + # with self.assertWarns(DeprecationWarning): + self.assertTrue(re.match('^(?i)' + upper_char, lower_char)) + # with self.assertWarns(DeprecationWarning): + self.assertTrue(re.match('$|(?i)' + upper_char, lower_char)) + # with self.assertWarns(DeprecationWarning) as warns: + self.assertTrue(re.match('(?:(?i)' + upper_char + ')', lower_char)) + # self.assertRegex(str(warns.warnings[0].message), + # 'Flags not at the start') + # self.assertEqual(warns.warnings[0].filename, __file__) + # with self.assertWarns(DeprecationWarning) as warns: + # self.assertTrue(re.fullmatch('(^)?(?(1)(?i)' + upper_char + ')', + # lower_char)) + # self.assertRegex(str(warns.warnings[0].message), + # 'Flags not at the start') + # self.assertEqual(warns.warnings[0].filename, __file__) + # with self.assertWarns(DeprecationWarning) as warns: + # self.assertTrue(re.fullmatch('($)?(?(1)|(?i)' + upper_char + ')', + # lower_char)) + # self.assertRegex(str(warns.warnings[0].message), + # 'Flags not at the start') + # self.assertEqual(warns.warnings[0].filename, __file__) + + + def test_dollar_matches_twice(self): + "$ matches the end of string, and just before the terminating \n" + pattern = re.compile('$') + self.assertEqual(pattern.sub('#', 'a\nb\n'), 'a\nb#\n#') + self.assertEqual(pattern.sub('#', 'a\nb\nc'), 'a\nb\nc#') + self.assertEqual(pattern.sub('#', '\n'), '#\n#') + + pattern = re.compile('$', re.MULTILINE) + self.assertEqual(pattern.sub('#', 'a\nb\n' ), 'a#\nb#\n#' ) + self.assertEqual(pattern.sub('#', 'a\nb\nc'), 'a#\nb#\nc#') + self.assertEqual(pattern.sub('#', '\n'), '#\n#') + + # def test_bytes_str_mixing(self): + # # Mixing str and bytes is disallowed + # pat = re.compile('.') + # bpat = re.compile(b'.') + # self.assertRaises(TypeError, pat.match, b'b') + # self.assertRaises(TypeError, bpat.match, 'b') + # self.assertRaises(TypeError, pat.sub, b'b', 'c') + # self.assertRaises(TypeError, pat.sub, 'b', b'c') + # self.assertRaises(TypeError, pat.sub, b'b', b'c') + # self.assertRaises(TypeError, bpat.sub, b'b', 'c') + # self.assertRaises(TypeError, bpat.sub, 'b', b'c') + # self.assertRaises(TypeError, bpat.sub, 'b', 'c') + + def test_ascii_and_unicode_flag(self): + # String patterns + for flags in (0, re.UNICODE): + pat = re.compile('\xc0', flags | re.IGNORECASE) + self.assertTrue(pat.match('\xe0')) + # pat = re.compile(r'\w', flags) # in javascript \w does not change in unicode mode + # self.assertTrue(pat.match('\xe0')) + # pat = re.compile('\xc0', re.ASCII | re.IGNORECASE) + # self.assertIsNone(pat.match('\xe0')) + # pat = re.compile('(?a)\xc0', re.IGNORECASE) + # self.assertIsNone(pat.match('\xe0')) + pat = re.compile(r'\w', re.ASCII) + self.assertIsNone(pat.match('\xe0')) + pat = re.compile(r'(?a)\w') + self.assertIsNone(pat.match('\xe0')) + # # Bytes patterns + # for flags in (0, re.ASCII): + # pat = re.compile(b'\xc0', flags | re.IGNORECASE) + # self.assertIsNone(pat.match(b'\xe0')) + # pat = re.compile(br'\w', flags) + # self.assertIsNone(pat.match(b'\xe0')) + # Incompatibilities + # self.assertRaises(ValueError, re.compile, br'\w', re.UNICODE) + # self.assertRaises(re.error, re.compile, br'(?u)\w') + self.assertRaises(ValueError, re.compile, r'\w', re.UNICODE | re.ASCII) + self.assertRaises(ValueError, re.compile, r'(?u)\w', re.ASCII) + self.assertRaises(ValueError, re.compile, r'(?a)\w', re.UNICODE) + self.assertRaises(re.error, re.compile, r'(?au)\w') + + # def test_locale_flag(self): + # enc = locale.getpreferredencoding() + # # Search non-ASCII letter + # for i in range(128, 256): + # try: + # c = bytes([i]).decode(enc) + # sletter = c.lower() + # if sletter == c: continue + # bletter = sletter.encode(enc) + # if len(bletter) != 1: continue + # if bletter.decode(enc) != sletter: continue + # bpat = re.escape(bytes([i])) + # break + # except (UnicodeError, TypeError): + # pass + # else: + # bletter = None + # bpat = b'A' + # # Bytes patterns + # pat = re.compile(bpat, re.LOCALE | re.IGNORECASE) + # if bletter: + # self.assertTrue(pat.match(bletter)) + # pat = re.compile(b'(?L)' + bpat, re.IGNORECASE) + # if bletter: + # self.assertTrue(pat.match(bletter)) + # pat = re.compile(bpat, re.IGNORECASE) + # if bletter: + # self.assertIsNone(pat.match(bletter)) + # pat = re.compile(br'\w', re.LOCALE) + # if bletter: + # self.assertTrue(pat.match(bletter)) + # pat = re.compile(br'(?L)\w') + # if bletter: + # self.assertTrue(pat.match(bletter)) + # pat = re.compile(br'\w') + # if bletter: + # self.assertIsNone(pat.match(bletter)) + # # Incompatibilities + # self.assertRaises(ValueError, re.compile, '', re.LOCALE) + # self.assertRaises(re.error, re.compile, '(?L)') + # self.assertRaises(ValueError, re.compile, b'', re.LOCALE | re.ASCII) + # self.assertRaises(ValueError, re.compile, b'(?L)', re.ASCII) + # self.assertRaises(ValueError, re.compile, b'(?a)', re.LOCALE) + # self.assertRaises(re.error, re.compile, b'(?aL)') + + def test_scoped_flags(self): + # javascript does not support inline modifiers leave failing tests here but the error messages won't work + # self.assertTrue(re.match(r'(?i:a)b', 'Ab')) + # self.assertIsNone(re.match(r'(?i:a)b', 'aB')) + # self.assertIsNone(re.match(r'(?-i:a)b', 'Ab', re.IGNORECASE)) + # self.assertTrue(re.match(r'(?-i:a)b', 'aB', re.IGNORECASE)) + # self.assertIsNone(re.match(r'(?i:(?-i:a)b)', 'Ab')) + # self.assertTrue(re.match(r'(?i:(?-i:a)b)', 'aB')) + + # self.assertTrue(re.match(r'(?x: a) b', 'a b')) + # self.assertIsNone(re.match(r'(?x: a) b', ' a b')) + # self.assertTrue(re.match(r'(?-x: a) b', ' ab', re.VERBOSE)) + # self.assertIsNone(re.match(r'(?-x: a) b', 'ab', re.VERBOSE)) + + # self.assertTrue(re.match(r'\w(?a:\W)\w', '\xe0\xe0\xe0')) + # self.assertTrue(re.match(r'(?a:\W(?u:\w)\W)', '\xe0\xe0\xe0')) + # self.assertTrue(re.match(r'\W(?u:\w)\W', '\xe0\xe0\xe0', re.ASCII)) + + self.checkPatternError(r'(?a)(?-a:\w)', + "bad inline flags: cannot turn off flags 'a', 'u' and 'L'", 8) + self.checkPatternError(r'(?i-i:a)', + 'bad inline flags: flag turned on and off', 5) + self.checkPatternError(r'(?au:a)', + "bad inline flags: flags 'a', 'u' and 'L' are incompatible", 4) + # self.checkPatternError(br'(?aL:a)', + # "bad inline flags: flags 'a', 'u' and 'L' are incompatible", 4) + + self.checkPatternError(r'(?-', 'missing flag', 3) + self.checkPatternError(r'(?-+', 'missing flag', 3) + self.checkPatternError(r'(?-z', 'unknown flag', 3) + self.checkPatternError(r'(?-i', 'missing :', 4) + self.checkPatternError(r'(?-i)', 'missing :', 4) + self.checkPatternError(r'(?-i+', 'missing :', 4) + self.checkPatternError(r'(?-iz', 'unknown flag', 4) + self.checkPatternError(r'(?i:', 'missing ), unterminated subpattern', 0) + self.checkPatternError(r'(?i', 'missing -, : or )', 3) + self.checkPatternError(r'(?i+', 'missing -, : or )', 3) + self.checkPatternError(r'(?iz', 'unknown flag', 3) + + def test_bug_6509(self): + # Replacement strings of both types must parse properly. + # all strings + pat = re.compile(r'a(\w)') + self.assertEqual(pat.sub('b\\1', 'ac'), 'bc') + pat = re.compile('a(.)') + self.assertEqual(pat.sub('b\\1', 'a\u1234'), 'b\u1234') + pat = re.compile('..') + self.assertEqual(pat.sub(lambda m: 'str', 'a5'), 'str') + + # # all bytes + # pat = re.compile(br'a(\w)') + # self.assertEqual(pat.sub(b'b\\1', b'ac'), b'bc') + # pat = re.compile(b'a(.)') + # self.assertEqual(pat.sub(b'b\\1', b'a\xCD'), b'b\xCD') + # pat = re.compile(b'..') + # self.assertEqual(pat.sub(lambda m: b'bytes', b'a5'), b'bytes') + + def test_dealloc(self): + # issue 3299: check for segfault in debug build + # import _sre + # the overflow limit is different on wide and narrow builds and it + # depends on the definition of SRE_CODE (see sre.h). + # 2**128 should be big enough to overflow on both. For smaller values + # a RuntimeError is raised instead of OverflowError. + long_overflow = 2**128 + self.assertRaises(TypeError, re.finditer, "a", {}) + # with self.assertRaises(OverflowError): + # _sre.compile("abc", 0, [long_overflow], 0, {}, ()) + # with self.assertRaises(TypeError): + # _sre.compile({}, 0, [], 0, [], []) + + def test_search_dot_unicode(self): + self.assertTrue(re.search("123.*-", '123abc-')) + self.assertTrue(re.search("123.*-", '123\xe9-')) + self.assertTrue(re.search("123.*-", '123\u20ac-')) + self.assertTrue(re.search("123.*-", '123\U0010ffff-')) + self.assertTrue(re.search("123.*-", '123\xe9\u20ac\U0010ffff-')) + + def test_compile(self): + # Test return value when given string and pattern as parameter + pattern = re.compile('random pattern') + self.assertIsInstance(pattern, re.Pattern) + same_pattern = re.compile(pattern) + self.assertIsInstance(same_pattern, re.Pattern) + self.assertIs(same_pattern, pattern) + # Test behaviour when not given a string or pattern as parameter + self.assertRaises(TypeError, re.compile, 0) + + # @bigmemtest(size=_2G, memuse=1) + # def test_large_search(self, size): + # # Issue #10182: indices were 32-bit-truncated. + # s = 'a' * size + # m = re.search('$', s) + # self.assertIsNotNone(m) + # self.assertEqual(m.start(), size) + # self.assertEqual(m.end(), size) + + # The huge memuse is because of re.sub() using a list and a join() + # to create the replacement result. + # @bigmemtest(size=_2G, memuse=16 + 2) + # def test_large_subn(self, size): + # # Issue #10182: indices were 32-bit-truncated. + # s = 'a' * size + # r, n = re.subn('', '', s) + # self.assertEqual(r, s) + # self.assertEqual(n, size + 1) + + def test_bug_16688(self): + # Issue 16688: Backreferences make case-insensitive regex fail on + # non-ASCII strings. + self.assertEqual(re.findall(r"(?i)(a)\1", "aa \u0100"), ['a']) + self.assertEqual(re.match(r"(?s).{1,3}", "\u0100\u0100").span(), (0, 2)) + + def test_repeat_minmax_overflow(self): + # Issue #13169 + string = "x" * 100000 + self.assertEqual(re.match(r".{65535}", string).span(), (0, 65535)) + self.assertEqual(re.match(r".{,65535}", string).span(), (0, 65535)) + self.assertEqual(re.match(r".{65535,}?", string).span(), (0, 65535)) + self.assertEqual(re.match(r".{65536}", string).span(), (0, 65536)) + self.assertEqual(re.match(r".{,65536}", string).span(), (0, 65536)) + self.assertEqual(re.match(r".{65536,}?", string).span(), (0, 65536)) + # 2**128 should be big enough to overflow both SRE_CODE and Py_ssize_t. + # self.assertRaises(OverflowError, re.compile, r".{%d}" % 2**128) + # self.assertRaises(OverflowError, re.compile, r".{,%d}" % 2**128) + # self.assertRaises(OverflowError, re.compile, r".{%d,}?" % 2**128) + # self.assertRaises(OverflowError, re.compile, r".{%d,%d}" % (2**129, 2**128)) + + # @cpython_only + # def test_repeat_minmax_overflow_maxrepeat(self): + # try: + # from _sre import MAXREPEAT + # except ImportError: + # self.skipTest('requires _sre.MAXREPEAT constant') + # string = "x" * 100000 + # self.assertIsNone(re.match(r".{%d}" % (MAXREPEAT - 1), string)) + # self.assertEqual(re.match(r".{,%d}" % (MAXREPEAT - 1), string).span(), + # (0, 100000)) + # self.assertIsNone(re.match(r".{%d,}?" % (MAXREPEAT - 1), string)) + # self.assertRaises(OverflowError, re.compile, r".{%d}" % MAXREPEAT) + # self.assertRaises(OverflowError, re.compile, r".{,%d}" % MAXREPEAT) + # self.assertRaises(OverflowError, re.compile, r".{%d,}?" % MAXREPEAT) + + def test_backref_group_name_in_exception(self): + # Issue 17341: Poor error message when compiling invalid regex + self.checkPatternError('(?P=)', + "bad character in group name ''", 4) + + def test_group_name_in_exception(self): + # Issue 17341: Poor error message when compiling invalid regex + self.checkPatternError('(?P)', + "bad character in group name '?foo'", 4) + + def test_issue17998(self): + for reps in '*', '+', '?', '{1}': + for mod in '', '?': + pattern = '.' + reps + mod + 'yz' + self.assertEqual(re.compile(pattern, re.S).findall('xyz'), + ['xyz']) + # pattern = pattern.encode() + # self.assertEqual(re.compile(pattern, re.S).findall(b'xyz'), + # [b'xyz'], msg=pattern) + + def test_match_repr(self): + for string in '[abracadabra]', S('[abracadabra]'): + m = re.search(r'(.+)(.*?)\1', string) + pattern = r"<(%s\.)?%s object; span=\(1, 12\), match='abracadabra'>" % ( + type(m).__module__, type(m).__name__ + ) + self.assertTrue(re.fullmatch(pattern, repr(m))) + # self.assertRegex(repr(m), pattern) + + # for string in (b'[abracadabra]', B(b'[abracadabra]'), + # bytearray(b'[abracadabra]'), + # memoryview(b'[abracadabra]')): + # m = re.search(br'(.+)(.*?)\1', string) + # pattern = r"<(%s\.)?%s object; span=\(1, 12\), match=b'abracadabra'>" % ( + # type(m).__module__, type(m).__qualname__ + # ) + # self.assertRegex(repr(m), pattern) + + first, second = list(re.finditer("(aa)|(bb)", "aa bb")) + pattern = r"<(%s\.)?%s object; span=\(0, 2\), match='aa'>" % ( + type(second).__module__, type(second).__name__ + ) + self.assertTrue(re.fullmatch(pattern, repr(first))) + # self.assertRegex(repr(first), pattern) + pattern = r"<(%s\.)?%s object; span=\(3, 5\), match='bb'>" % ( + type(second).__module__, type(second).__name__ + ) + self.assertTrue(re.fullmatch(pattern, repr(second))) + # self.assertRegex(repr(second), pattern) + + def test_zerowidth(self): + # these don't work in skulpt + # Issues 852532, 1647489, 3262, 25054. + self.assertEqual(re.split(r"\b", "a::bc"), ['', 'a', '::', 'bc', '']) + # self.assertEqual(re.split(r"\b|:+", "a::bc"), ['', 'a', '', '', 'bc', '']) + self.assertEqual(re.split(r"(?)', 'unknown extension ?<>', 1) + self.checkPatternError(r'(?', 'unexpected end of pattern', 2) + + def test_enum(self): + # Issue #28082: Check that str(flag) returns a human readable string + # instead of an integer + self.assertIn('ASCII', str(re.A)) + self.assertIn('DOTALL', str(re.S)) + + def test_pattern_compare(self): + pattern1 = re.compile('abc', re.IGNORECASE) + + # equal to itself + self.assertEqual(pattern1, pattern1) + self.assertFalse(pattern1 != pattern1) + + # equal + re.purge() + pattern2 = re.compile('abc', re.IGNORECASE) + self.assertEqual(hash(pattern2), hash(pattern1)) + self.assertEqual(pattern2, pattern1) + + # not equal: different pattern + re.purge() + pattern3 = re.compile('XYZ', re.IGNORECASE) + # Don't test hash(pattern3) != hash(pattern1) because there is no + # warranty that hash values are different + self.assertNotEqual(pattern3, pattern1) + + # not equal: different flag (flags=0) + re.purge() + pattern4 = re.compile('abc') + self.assertNotEqual(pattern4, pattern1) + + # only == and != comparison operators are supported + with self.assertRaises(TypeError): + pattern1 < pattern2 + + # def test_pattern_compare_bytes(self): + # pattern1 = re.compile(b'abc') + + # # equal: test bytes patterns + # re.purge() + # pattern2 = re.compile(b'abc') + # self.assertEqual(hash(pattern2), hash(pattern1)) + # self.assertEqual(pattern2, pattern1) + + # not equal: pattern of a different types (str vs bytes), + # comparison must not raise a BytesWarning + # re.purge() + # pattern3 = re.compile('abc') + # with warnings.catch_warnings(): + # warnings.simplefilter('error', BytesWarning) + # self.assertNotEqual(pattern3, pattern1) + + # def test_bug_29444(self): + # s = bytearray(b'abcdefgh') + # m = re.search(b'[a-h]+', s) + # m2 = re.search(b'[e-h]+', s) + # self.assertEqual(m.group(), b'abcdefgh') + # self.assertEqual(m2.group(), b'efgh') + # s[:] = b'xyz' + # self.assertEqual(m.group(), b'xyz') + # self.assertEqual(m2.group(), b'') + + def test_bug_34294(self): + # Issue 34294: wrong capturing groups + + # exists since Python 2 + s = "a\tx" + p = r"\b(?=(\t)|(x))x" + self.assertEqual(re.search(p, s).groups(), (None, 'x')) + + # introduced in Python 3.7.0 + s = "ab" + p = r"(?=(.)(.)?)" + self.assertEqual(re.findall(p, s), + [('a', 'b'), ('b', '')]) + self.assertEqual([m.groups() for m in re.finditer(p, s)], + [('a', 'b'), ('b', None)]) + + # test-cases provided by issue34294, introduced in Python 3.7.0 + p = r"(?=<(?P\w+)/?>(?:(?P.+?))?)" + s = "" + self.assertEqual(re.findall(p, s), + [('test', ''), ('foo2', '')]) + self.assertEqual([m.groupdict() for m in re.finditer(p, s)], + [{'tag': 'test', 'text': ''}, + {'tag': 'foo2', 'text': None}]) + s = "Hello" + self.assertEqual([m.groupdict() for m in re.finditer(p, s)], + [{'tag': 'test', 'text': 'Hello'}, + {'tag': 'foo', 'text': None}]) + s = "Hello" + self.assertEqual([m.groupdict() for m in re.finditer(p, s)], + [{'tag': 'test', 'text': 'Hello'}, + {'tag': 'foo', 'text': None}, + {'tag': 'foo', 'text': None}]) + + +class PatternReprTests(unittest.TestCase): + def check(self, pattern, expected): + self.assertEqual(repr(re.compile(pattern)), expected) + + def check_flags(self, pattern, flags, expected): + self.assertEqual(repr(re.compile(pattern, flags)), expected) + + def test_without_flags(self): + self.check('random pattern', + "re.compile('random pattern')") + + def test_single_flag(self): + self.check_flags('random pattern', re.IGNORECASE, + "re.compile('random pattern', re.IGNORECASE)") + + def test_multiple_flags(self): + self.check_flags('random pattern', re.I|re.S|re.X, + "re.compile('random pattern', " + "re.IGNORECASE|re.DOTALL|re.VERBOSE)") + + def test_unicode_flag(self): + self.check_flags('random pattern', re.U, + "re.compile('random pattern')") + self.check_flags('random pattern', re.I|re.S|re.U, + "re.compile('random pattern', " + "re.IGNORECASE|re.DOTALL)") + + def test_inline_flags(self): + self.check('(?i)pattern', + "re.compile('(?i)pattern', re.IGNORECASE)") + + def test_unknown_flags(self): + self.check_flags('random pattern', 0x123000, + "re.compile('random pattern', 0x123000)") + self.check_flags('random pattern', 0x123000|re.I, + "re.compile('random pattern', re.IGNORECASE|0x123000)") + + # def test_bytes(self): + # self.check(b'bytes pattern', + # "re.compile(b'bytes pattern')") + # self.check_flags(b'bytes pattern', re.A, + # "re.compile(b'bytes pattern', re.ASCII)") + + # def test_locale(self): + # self.check_flags(b'bytes pattern', re.L, + # "re.compile(b'bytes pattern', re.LOCALE)") + + def test_quotes(self): + self.check('random "double quoted" pattern', + '''re.compile('random "double quoted" pattern')''') + self.check("random 'single quoted' pattern", + '''re.compile("random 'single quoted' pattern")''') + self.check('''both 'single' and "double" quotes''', + '''re.compile('both \\'single\\' and "double" quotes')''') + + def test_long_pattern(self): + pattern = 'Very %spattern' % ('long ' * 1000) + r = repr(re.compile(pattern)) + self.assertLess(len(r), 300) + self.assertEqual(r[:30], "re.compile('Very long long lon") + r = repr(re.compile(pattern, re.I)) + self.assertLess(len(r), 300) + self.assertEqual(r[:30], "re.compile('Very long long lon") + self.assertEqual(r[-16:], ", re.IGNORECASE)") + + def test_flags_repr(self): + self.assertEqual(repr(re.I), "re.IGNORECASE") + self.assertEqual(repr(re.I|re.S|re.X), + "re.IGNORECASE|re.DOTALL|re.VERBOSE") + self.assertEqual(repr(re.I|re.S|re.X|(1<<20)), + "re.IGNORECASE|re.DOTALL|re.VERBOSE|0x100000") + self.assertEqual(repr(~re.I), "~re.IGNORECASE") + self.assertEqual(repr(~(re.I|re.S|re.X)), + "~(re.IGNORECASE|re.DOTALL|re.VERBOSE)") + self.assertEqual(repr(~(re.I|re.S|re.X|(1<<20))), + "~(re.IGNORECASE|re.DOTALL|re.VERBOSE|0x100000)") + + +# class ImplementationTest(unittest.TestCase): +# """ +# Test implementation details of the re module. +# """ +# pass + +# def test_overlap_table(self): +# f = sre_compile._generate_overlap_table +# self.assertEqual(f(""), []) +# self.assertEqual(f("a"), [0]) +# self.assertEqual(f("abcd"), [0, 0, 0, 0]) +# self.assertEqual(f("aaaa"), [0, 1, 2, 3]) +# self.assertEqual(f("ababba"), [0, 0, 1, 2, 0, 1]) +# self.assertEqual(f("abcabdac"), [0, 0, 0, 1, 2, 0, 1, 0]) + + +# class ExternalTests(unittest.TestCase): + +# def test_re_benchmarks(self): +# 're_tests benchmarks' +# from test.re_tests import benchmarks +# for pattern, s in benchmarks: +# with self.subTest(pattern=pattern, string=s): +# p = re.compile(pattern) +# self.assertTrue(p.search(s)) +# self.assertTrue(p.match(s)) +# self.assertTrue(p.fullmatch(s)) +# s2 = ' '*10000 + s + ' '*10000 +# self.assertTrue(p.search(s2)) +# self.assertTrue(p.match(s2, 10000)) +# self.assertTrue(p.match(s2, 10000, 10000 + len(s))) +# self.assertTrue(p.fullmatch(s2, 10000, 10000 + len(s))) + +# def test_re_tests(self): +# 're_tests test suite' +# from test.re_tests import tests, SUCCEED, FAIL, SYNTAX_ERROR +# for t in tests: +# pattern = s = outcome = repl = expected = None +# if len(t) == 5: +# pattern, s, outcome, repl, expected = t +# elif len(t) == 3: +# pattern, s, outcome = t +# else: +# raise ValueError('Test tuples should have 3 or 5 fields', t) + +# with self.subTest(pattern=pattern, string=s): +# if outcome == SYNTAX_ERROR: # Expected a syntax error +# with self.assertRaises(re.error): +# re.compile(pattern) +# continue + +# obj = re.compile(pattern) +# result = obj.search(s) +# if outcome == FAIL: +# self.assertIsNone(result, 'Succeeded incorrectly') +# continue + +# with self.subTest(): +# self.assertTrue(result, 'Failed incorrectly') +# # Matched, as expected, so now we compute the +# # result string and compare it to our expected result. +# start, end = result.span(0) +# vardict = {'found': result.group(0), +# 'groups': result.group(), +# 'flags': result.re.flags} +# for i in range(1, 100): +# try: +# gi = result.group(i) +# # Special hack because else the string concat fails: +# if gi is None: +# gi = "None" +# except IndexError: +# gi = "Error" +# vardict['g%d' % i] = gi +# for i in result.re.groupindex.keys(): +# try: +# gi = result.group(i) +# if gi is None: +# gi = "None" +# except IndexError: +# gi = "Error" +# vardict[i] = gi +# self.assertEqual(eval(repl, vardict), expected, +# 'grouping error') + +# # Try the match with both pattern and string converted to +# # bytes, and check that it still succeeds. +# try: +# bpat = bytes(pattern, "ascii") +# bs = bytes(s, "ascii") +# except UnicodeEncodeError: +# # skip non-ascii tests +# pass +# else: +# with self.subTest('bytes pattern match'): +# obj = re.compile(bpat) +# self.assertTrue(obj.search(bs)) + +# # Try the match with LOCALE enabled, and check that it +# # still succeeds. +# with self.subTest('locale-sensitive match'): +# obj = re.compile(bpat, re.LOCALE) +# result = obj.search(bs) +# if result is None: +# print('=== Fails on locale-sensitive match', t) + +# # Try the match with the search area limited to the extent +# # of the match and see if it still succeeds. \B will +# # break (because it won't match at the end or start of a +# # string), so we'll ignore patterns that feature it. +# if (pattern[:2] != r'\B' and pattern[-2:] != r'\B' +# and result is not None): +# with self.subTest('range-limited match'): +# obj = re.compile(pattern) +# self.assertTrue(obj.search(s, start, end + 1)) + +# # Try the match with IGNORECASE enabled, and check that it +# # still succeeds. +# with self.subTest('case-insensitive match'): +# obj = re.compile(pattern, re.IGNORECASE) +# self.assertTrue(obj.search(s)) + +# # Try the match with UNICODE locale enabled, and check +# # that it still succeeds. +# with self.subTest('unicode-sensitive match'): +# obj = re.compile(pattern, re.UNICODE) +# self.assertTrue(obj.search(s)) + + +######## test methods ######""" Unit test for re module""" + +class ReTestsBasic(unittest.TestCase): def test_findall(self): #Skulpt is failing all the commented out tests in test_findall and it shouldn't be val = re.findall("From","dlkjdsljkdlkdsjlk") @@ -284,9 +2633,8 @@ def test_split(self): self.assertEqual(re.split("\W+", "Words, words, words.", 1), ['Words', 'words, words.']) self.assertEqual(re.split('[a-f]+', '0a3B9', 0, re.IGNORECASE), ['0', '3', '9']) self.assertEqual(re.split("(\W+)", '...words, words...'), ['', '...', 'words', ', ', 'words', '...', '']) - #Skulpt fails the test below and it shouldn't - #self.assertEqual(re.split('x*', 'foo'), ['', 'f', 'o', 'o', '']) + self.assertEqual(re.split('x*', 'foo'), ['', 'f', 'o', 'o', '']) -if __name__ == '__main__': - unittest.main() +if __name__ == "__main__": + unittest.main(verbosity=2) \ No newline at end of file From 877c0bcfe34be954dc7f55516cb8277c295072dc Mon Sep 17 00:00:00 2001 From: mrcork Date: Mon, 5 Oct 2020 22:03:41 +0800 Subject: [PATCH 113/137] adjust inline flags --- src/lib/re.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/re.js b/src/lib/re.js index 5fc524466c..93b8f36452 100644 --- a/src/lib/re.js +++ b/src/lib/re.js @@ -204,7 +204,7 @@ function $builtinmodule(name) { "cannot use LOCALE flag with a str pattern": re.L, "ASCII and UNICODE flags are incompatible": new re.RegexFlag(re.A.valueOf() | re.U.valueOf()), }); - const inline_regex = /\(\?([i|s|a|m|u|x]+)\)/g; + const inline_regex = /\(\?([isamux]+)\)/g; function adjustFlags(pyPattern, pyFlag) { let jsPattern = pyPattern.toString(); From 5804fd93dd11a632e422dfbc54b7fc49fb4d35f2 Mon Sep 17 00:00:00 2001 From: mrcork Date: Sun, 4 Oct 2020 00:45:25 +0800 Subject: [PATCH 114/137] allow mapping proxy to be called internally --- src/mappingproxy.js | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/mappingproxy.js b/src/mappingproxy.js index 69a3e0eaa1..cde88789f9 100644 --- a/src/mappingproxy.js +++ b/src/mappingproxy.js @@ -31,12 +31,20 @@ Sk.builtin.mappingproxy = Sk.abstr.buildNativeClass("mappingproxy", { constructor: function mappingproxy(d) { Sk.asserts.assert(this instanceof Sk.builtin.mappingproxy, "bad call to mapping proxy, use 'new'"); - this.mapping = new Sk.builtin.dict([]); if (d !== undefined) { - // internal call so d is an object literal - // adust this.mapping.entries to be a custom getter - // allowing support for dynamic object literals - customEntriesGetter(this.mapping, d); + const constructor = d.constructor; + if (constructor === Object || constructor === null || d.hasOwnProperty("sk$object")) { + // then we are not a mapping but js object to be proxied + this.mapping = new Sk.builtin.dict([]); + // internal call so d is an object literal + // adust this.mapping.entries to be a custom getter + // allowing support for dynamic object literals + customEntriesGetter(this.mapping, d); + } else if (Sk.builtin.checkMapping(d)) { + this.mapping = d; + } else { + Sk.asserts.fail("unhandled case for mappingproxy"); + } } }, slots: { From 163f7257c4ff38341cb05c3dbc8b9df5de4bf0ec Mon Sep 17 00:00:00 2001 From: mrcork Date: Sun, 4 Oct 2020 00:43:11 +0800 Subject: [PATCH 115/137] put bigup on JSBI - todo use this elsewhere --- support/polyfills/JSBI.js | 1 + 1 file changed, 1 insertion(+) diff --git a/support/polyfills/JSBI.js b/support/polyfills/JSBI.js index 1f9afaf379..d85e11dce9 100644 --- a/support/polyfills/JSBI.js +++ b/support/polyfills/JSBI.js @@ -90,3 +90,4 @@ JSBI.__ZERO = JSBI.BigInt(0); JSBI.__MAX_SAFE = JSBI.BigInt(Number.MAX_SAFE_INTEGER); JSBI.__MIN_SAFE = JSBI.BigInt(-Number.MAX_SAFE_INTEGER); JSBI.numberIfSafe = (val) => (JSBI.lessThan(val, JSBI.__MAX_SAFE) && JSBI.greaterThan(val, JSBI.__MIN_SAFE) ? JSBI.toNumber(val) : val); +JSBI.BigUp = (val) => JSBI.__isBigInt(val) ? val : JSBI.BigInt(val); \ No newline at end of file From 3cde5666806f14439f024048084d2e17b0a051af Mon Sep 17 00:00:00 2001 From: mrcork Date: Fri, 19 Feb 2021 09:35:30 +0800 Subject: [PATCH 116/137] remove debugger --- src/lib/re.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/lib/re.js b/src/lib/re.js index 93b8f36452..37ef6c5f25 100644 --- a/src/lib/re.js +++ b/src/lib/re.js @@ -304,7 +304,6 @@ function $builtinmodule(name) { let regex; let msg; let unicodeEscapedPattern = jsPattern; - debugger; if (jsFlags.includes("u")) { // then we we need to adjust the escapes for \\\t to be \\t etc because javascript reads escapes differently in unicode mode! // '\\-' is different - inside a square bracket it gets compiled but outside it doesn't! @@ -315,7 +314,6 @@ function $builtinmodule(name) { case "\\\t": return "\\t"; case "\\\n": - debugger; return "\\n"; case "\\\v": return "\\v"; From 19a8b49c8e6f8ca59731664209787161354803a8 Mon Sep 17 00:00:00 2001 From: mrcork Date: Fri, 19 Feb 2021 09:36:13 +0800 Subject: [PATCH 117/137] make sure that matchAll gets polyfilled by updating webpack/closure --- support/polyfills/es6.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/support/polyfills/es6.js b/support/polyfills/es6.js index ef96bc2f33..38bb48ada6 100644 --- a/support/polyfills/es6.js +++ b/support/polyfills/es6.js @@ -1,4 +1,5 @@ // stdlib es6 polyfills // make sure closure compiler knows about these functions so that it can polyfill them -[...'abc'].flat(); \ No newline at end of file +[...'abc'].flat(); +"a".matchAll(/a/g); From 8244702158758b9df40901e5fe6d913cfeeb7122 Mon Sep 17 00:00:00 2001 From: Stu Cork Date: Fri, 13 Aug 2021 20:38:22 +0800 Subject: [PATCH 118/137] deconstruct Sk object and use prototype.sticky rather than try catch --- src/lib/re.js | 267 +++++++++++++++++++++++++++----------------------- 1 file changed, 145 insertions(+), 122 deletions(-) diff --git a/src/lib/re.js b/src/lib/re.js index 37ef6c5f25..14e21ee671 100644 --- a/src/lib/re.js +++ b/src/lib/re.js @@ -1,7 +1,36 @@ function $builtinmodule(name) { + + const { + builtin: { + dict: pyDict, + str: pyStr, + list: pyList, + int_: pyInt, + type: pyType, + tuple: pyTuple, + mappingproxy: pyMappingProxy, + slice: pySlice, + none: {none$: pyNone}, + NotImplemented: { NotImplemented$: pyNotImplemented }, + Exception, + OverflowError, + IndexError, + TypeError, + ValueError, + checkInt, + checkString, + checkCallable, + hex, + }, + abstr: { buildNativeClass, typeName, checkOneArg, numberBinOp, copyKeywordToNamedArgs, setUpModuleMethods }, + misceval: { iterator: pyIterator, objectRepr, asIndexSized, isIndex, callsimArray: pyCall }, + } = Sk; + + + const re = { - __name__: new Sk.builtin.str("re"), - __all__: new Sk.builtin.list( + __name__: new pyStr("re"), + __all__: new pyList( [ "match", "fullmatch", @@ -32,25 +61,25 @@ function $builtinmodule(name) { "DOTALL", "VERBOSE", "UNICODE", - ].map((x) => new Sk.builtin.str(x)) + ].map((x) => new pyStr(x)) ), }; // cached flags const _value2member = {}; - const RegexFlagMeta = Sk.abstr.buildNativeClass("RegexFlagMeta", { + const RegexFlagMeta = buildNativeClass("RegexFlagMeta", { constructor: function RegexFlagMeta() {}, - base: Sk.builtin.type, + base: pyType, slots: { tp$iter() { const members = Object.values(_members)[Symbol.iterator](); - return new Sk.misceval.iterator(() => members.next().value); + return new pyIterator(() => members.next().value); }, sq$contains(flag) { if (!(flag instanceof this)) { - throw new Sk.builtin.TypeError( - "unsupported operand type(s) for 'in': '" + Sk.abstr.typeName(flag) + "' and '" + Sk.abstr.typeName(this) + "'" + throw new TypeError( + "unsupported operand type(s) for 'in': '" + typeName(flag) + "' and '" + typeName(this) + "'" ); } return Object.values(_members).includes(flag); @@ -58,9 +87,9 @@ function $builtinmodule(name) { }, }); - re.RegexFlag = Sk.abstr.buildNativeClass("RegexFlag", { + re.RegexFlag = buildNativeClass("RegexFlag", { meta: RegexFlagMeta, - base: Sk.builtin.int_, + base: pyInt, constructor: function RegexFlag(value) { const member = _value2member[value]; if (member) { @@ -72,10 +101,10 @@ function $builtinmodule(name) { slots: { tp$new(args, kwargs) { - Sk.abstr.checkOneArg("RegexFlag", args, kwargs); + checkOneArg("RegexFlag", args, kwargs); const value = args[0].valueOf(); - if (!Sk.builtin.checkInt(value)) { - throw new Sk.builtin.ValueError(Sk.abstr.objectRepr(value) + " is not a valid RegexFlag"); + if (!checkInt(value)) { + throw new ValueError(objectRepr(value) + " is not a valid RegexFlag"); } return new re.RegexFlag(value); }, @@ -93,18 +122,18 @@ function $builtinmodule(name) { } }); if (value) { - members.push(Sk.builtin.hex(value).toString()); + members.push(hex(value).toString()); } let res = members.join("|"); if (neg) { res = members.length > 1 ? "~(" + res + ")" : "~" + res; } - return new Sk.builtin.str(res); + return new pyStr(res); }, sq$contains(flag) { if (!(flag instanceof re.RegexFlag)) { - throw new Sk.builtin.TypeError("'in' requires a RegexFlag not " + Sk.abstr.typeName(flag)); + throw new TypeError("'in' requires a RegexFlag not " + typeName(flag)); } return this.nb$and(flag) === flag; }, @@ -153,7 +182,7 @@ function $builtinmodule(name) { function flagBitSlot(number_func, bigint_func) { return function (other) { - if (other instanceof re.RegexFlag || other instanceof Sk.builtin.int_) { + if (other instanceof re.RegexFlag || other instanceof pyInt) { let v = this.v; let w = other.v; if (typeof v === "number" && typeof w === "number") { @@ -167,7 +196,7 @@ function $builtinmodule(name) { w = JSBI.BigUp(w); return new re.RegexFlag(JSBI.numberIfSafe(bigint_func(v, w))); } - return Sk.builtin.NotImplemented.NotImplemented$; + return pyNotImplemented; }; } @@ -187,18 +216,12 @@ function $builtinmodule(name) { x: re.X, }; - (function checkJsFlags() { - try { - new RegExp("", "u"); - } catch { - delete jsFlags["u"]; - } - try { - new RegExp("", "s"); - } catch { - delete jsFlags["s"]; - } - })(); + if (!RegExp.prototype.hasOwnProperty("sticky")) { + delete jsFlags["s"]; + } + if (!RegExp.prototype.hasOwnProperty("unicode")) { + delete jsFlags["u"]; + } const flagFails = Object.entries({ "cannot use LOCALE flag with a str pattern": re.L, @@ -227,22 +250,22 @@ function $builtinmodule(name) { } }); - pyFlag = Sk.abstr.numberBinOp(new re.RegexFlag(inlineFlags), pyFlag, "BitOr"); + pyFlag = numberBinOp(new re.RegexFlag(inlineFlags), pyFlag, "BitOr"); // check compatibility of flags flagFails.forEach(([msg, flag]) => { - if (Sk.abstr.numberBinOp(flag, pyFlag, "BitAnd") === flag) { - throw new Sk.builtin.ValueError(msg); + if (numberBinOp(flag, pyFlag, "BitAnd") === flag) { + throw new ValueError(msg); } }); // use unicode? - if (Sk.abstr.numberBinOp(re.A, pyFlag, "BitAnd") !== re.A) { - pyFlag = Sk.abstr.numberBinOp(re.U, pyFlag, "BitOr"); + if (numberBinOp(re.A, pyFlag, "BitAnd") !== re.A) { + pyFlag = numberBinOp(re.U, pyFlag, "BitOr"); } Object.entries(jsFlags).forEach(([flag, reFlag]) => { - if (Sk.abstr.numberBinOp(reFlag, pyFlag, "BitAnd") === reFlag) { + if (numberBinOp(reFlag, pyFlag, "BitAnd") === reFlag) { jsFlag += flag; } }); @@ -295,7 +318,7 @@ function $builtinmodule(name) { return p0 + "(?<" + p3 + ">"; } if (!named_groups[p2]) { - throw new re.error("unknown group name " + p2 + " at position " + offset + 1, pyPattern, new Sk.builtin.int_(offset + 1)); + throw new re.error("unknown group name " + p2 + " at position " + offset + 1, pyPattern, new pyInt(offset + 1)); } return p0 + "\\k<" + p2 + ">"; } @@ -350,31 +373,31 @@ function $builtinmodule(name) { function _compile(pattern, flag) { if (pattern instanceof re.Pattern) { if (flag !== zero || flag.valueOf()) { - throw new Sk.builtin.ValueError("cannot process flags argument with compiled pattern"); + throw new ValueError("cannot process flags argument with compiled pattern"); } return pattern; } - if (!Sk.builtin.checkString(pattern)) { - throw new Sk.builtin.TypeError("first argument must be string or compiled pattern"); + if (!checkString(pattern)) { + throw new TypeError("first argument must be string or compiled pattern"); } return compile_pattern(pattern, flag); // compile the pattern to javascript Regex } - re.error = Sk.abstr.buildNativeClass("re.error", { - base: Sk.builtin.Exception, + re.error = buildNativeClass("re.error", { + base: Exception, constructor: function error(msg, pattern, pos) { this.$pattern = pattern; this.$msg = msg; - this.$pos = pos || Sk.builtin.none.none$; - Sk.builtin.Exception.call(this, msg); + this.$pos = pos || pyNone; + Exception.call(this, msg); }, slots: { tp$doc: "Exception raised for invalid regular expressions.\n\n Attributes:\n\n msg: The unformatted error message\n pattern: The regular expression pattern\n", tp$init(args, kwargs) { - const [msg, pattern, pos] = Sk.abstr.copyKeywordToNamedArgs("re.error", ["msg", "pattern", "pos"], args, kwargs, [ - Sk.builtin.none.none$, - Sk.builtin.none.none$, + const [msg, pattern, pos] = copyKeywordToNamedArgs("re.error", ["msg", "pattern", "pos"], args, kwargs, [ + pyNone, + pyNone, ]); this.$pattern = pattern; this.$pos = pos; @@ -400,10 +423,10 @@ function $builtinmodule(name) { }, }); - const zero = new Sk.builtin.int_(0); + const zero = new pyInt(0); const maxsize = Number.MAX_SAFE_INTEGER; - re.Pattern = Sk.abstr.buildNativeClass("re.Pattern", { + re.Pattern = buildNativeClass("re.Pattern", { constructor: function (regex, str, flags) { this.v = regex; this.str = str; @@ -413,13 +436,13 @@ function $builtinmodule(name) { }, slots: { $r() { - const patrepr = Sk.misceval.objectRepr(this.str).slice(0, 200); - const flagrepr = Sk.misceval.objectRepr(this.$flags.nb$and(re.U.nb$invert())); // re.U is not included in the repr here - return new Sk.builtin.str("re.compile(" + patrepr + (flagrepr ? ", " + flagrepr : "") + ")"); + const patrepr = objectRepr(this.str).slice(0, 200); + const flagrepr = objectRepr(this.$flags.nb$and(re.U.nb$invert())); // re.U is not included in the repr here + return new pyStr("re.compile(" + patrepr + (flagrepr ? ", " + flagrepr : "") + ")"); }, tp$richcompare(other, op) { if ((op !== "Eq" && op !== "NotEq") || !(other instanceof re.Pattern)) { - return Sk.builtin.NotImplemented.NotImplemented$; + return pyNotImplemented; } const res = this.str === other.str && this.$flags === other.$flags; return op === "Eq" ? res : !res; @@ -540,7 +563,7 @@ function $builtinmodule(name) { // we know we have a compiled expression so we just need to check matching brackets // bracket characters that are not inside [] not followed by ? but could be followed by ?P< const num_matches = (this.str.v.match(this.group$regex) || []).length; - this.$groups = new Sk.builtin.int_(num_matches); + this.$groups = new pyInt(num_matches); } return this.$groups; }, @@ -554,12 +577,12 @@ function $builtinmodule(name) { let i = 1; for (match of matches) { if (match[1]) { - arr.push(new Sk.builtin.str(match[1])); - arr.push(new Sk.builtin.int_(i)); + arr.push(new pyStr(match[1])); + arr.push(new pyInt(i)); } i++; } - this.$groupindex = new Sk.builtin.mappingproxy(new Sk.builtin.dict(arr)); + this.$groupindex = new pyMappingProxy(new pyDict(arr)); } return this.$groupindex; }, @@ -571,17 +594,17 @@ function $builtinmodule(name) { // if it's a group like (?P) then we need to capture the foo group$regex: /\((?!\?(?!P<).*)(?:\?P<([^\d\W]\w*)>)?(?![^\[]*\])/g, get$count(count) { - count = Sk.misceval.asIndexSized(count, Sk.builtin.OverflowError); + count = asIndexSized(count, OverflowError); return count ? count : Number.POSITIVE_INFINITY; }, get$jsstr(string, pos, endpos) { - if (!Sk.builtin.checkString(string)) { - throw new Sk.builtin.TypeError("expected string or bytes-like object"); + if (!checkString(string)) { + throw new TypeError("expected string or bytes-like object"); } if ((pos === zero && endpos === maxsize) || (pos === undefined && endpos === undefined)) { return { jsstr: string.toString(), pos: zero.valueOf(), endpos: string.sq$length() }; } - const { start, end } = Sk.builtin.slice.startEnd$wrt(string, pos, endpos); + const { start, end } = pySlice.startEnd$wrt(string, pos, endpos); return { jsstr: string.toString().slice(start, end), pos: start, endpos: end }; }, find$all(string, pos, endpos) { @@ -593,16 +616,16 @@ function $builtinmodule(name) { // do we have groups? ret.push( match.length === 1 - ? new Sk.builtin.str(match[0]) + ? new pyStr(match[0]) : match.length === 2 - ? new Sk.builtin.str(match[1]) - : new Sk.builtin.tuple(match.slice(1).map((x) => new Sk.builtin.str(x))) + ? new pyStr(match[1]) + : new pyTuple(match.slice(1).map((x) => new pyStr(x))) ); } - return new Sk.builtin.list(ret); + return new pyList(ret); }, $split(string, maxsplit) { - maxsplit = Sk.misceval.asIndexSized(maxsplit); + maxsplit = asIndexSized(maxsplit); maxsplit = maxsplit ? maxsplit : Number.POSITIVE_INFINITY; let { jsstr } = this.get$jsstr(string); const regex = this.v; @@ -611,9 +634,9 @@ function $builtinmodule(name) { let num_splits = 0; let idx = 0; while ((match = regex.exec(jsstr)) !== null && num_splits < maxsplit) { - split.push(new Sk.builtin.str(jsstr.substring(idx, match.index))); + split.push(new pyStr(jsstr.substring(idx, match.index))); if (match.length > 1) { - split.push(...match.slice(1).map((x) => (x === undefined ? Sk.builtin.none.none$ : new Sk.builtin.str(x)))); + split.push(...match.slice(1).map((x) => (x === undefined ? pyNone : new pyStr(x)))); } num_splits++; idx = regex.lastIndex; @@ -629,8 +652,8 @@ function $builtinmodule(name) { } } regex.lastIndex = 0; - split.push(new Sk.builtin.str(jsstr.slice(idx))); - return new Sk.builtin.list(split); + split.push(new pyStr(jsstr.slice(idx))); + return new pyList(split); }, match$from_repl(args, string, pos, endpos) { let match_like; @@ -649,11 +672,11 @@ function $builtinmodule(name) { do$sub(repl, string, count) { const { jsstr, pos, endpos } = this.get$jsstr(string); let matchRepl; - if (Sk.builtin.checkCallable(repl)) { + if (checkCallable(repl)) { matchRepl = (matchObj) => { - const rep = Sk.misceval.callsimArray(repl, [matchObj]); - if (!Sk.builtin.checkString(rep)) { - throw new Sk.builtin.TypeError("expected str instance, " + Sk.abstr.typeName(rep) + " found"); + const rep = pyCall(repl, [matchObj]); + if (!checkString(rep)) { + throw new TypeError("expected str instance, " + typeName(rep) + " found"); } return rep.toString(); }; @@ -671,21 +694,21 @@ function $builtinmodule(name) { const matchObj = this.match$from_repl(args, string, pos, endpos); return matchRepl(matchObj); }); - return [new Sk.builtin.str(ret), new Sk.builtin.int_(num_repl)]; + return [new pyStr(ret), new pyInt(num_repl)]; }, $sub(repl, string, count) { const [ret] = this.do$sub(repl, string, count); return ret; }, $subn(repl, string, count) { - return new Sk.builtin.tuple(this.do$sub(repl, string, count)); + return new pyTuple(this.do$sub(repl, string, count)); }, do$match(regex, string, pos, endpos) { let jsstr; ({ jsstr, pos, endpos } = this.get$jsstr(string, pos, endpos)); const match = jsstr.match(regex); if (match === null) { - return Sk.builtin.none.none$; + return pyNone; } return new re.Match(match, this, string, pos, endpos); }, @@ -711,7 +734,7 @@ function $builtinmodule(name) { let jsstr; ({ jsstr, pos, endpos } = this.get$jsstr(string, pos, endpos)); const matchIter = jsstr.matchAll(this.v); - return new Sk.misceval.iterator(() => { + return new pyIterator(() => { const match = matchIter.next().value; if (match === undefined) { return undefined; @@ -726,10 +749,10 @@ function $builtinmodule(name) { }, }); - re.Match = Sk.abstr.buildNativeClass("re.Match", { + re.Match = buildNativeClass("re.Match", { constructor: function (match, re, str, pos, endpos) { this.v = match; // javascript match object; - this.$match = new Sk.builtin.str(this.v[0]); + this.$match = new pyStr(this.v[0]); this.str = str; this.$re = re; this.$pos = pos; @@ -747,13 +770,13 @@ function $builtinmodule(name) { //e.g. let ret = ""; - return new Sk.builtin.str(ret); + ret += "match=" + objectRepr(this.$match) + ">"; + return new pyStr(ret); }, tp$as_squence_or_mapping: true, mp$subscript(item) { const ret = this.get$group(item); - return ret === undefined ? Sk.builtin.none.none$ : new Sk.builtin.str(ret); + return ret === undefined ? pyNone : new pyStr(ret); }, }, methods: { @@ -762,14 +785,14 @@ function $builtinmodule(name) { let ret; if (gs.length <= 1) { ret = this.get$group(gs[0]); - return ret === undefined ? Sk.builtin.none.none$ : new Sk.builtin.str(ret); + return ret === undefined ? pyNone : new pyStr(ret); } ret = []; gs.forEach((g) => { g = this.get$group(g); - ret.push(g === undefined ? Sk.builtin.none.none$ : new Sk.builtin.str(g)); + ret.push(g === undefined ? pyNone : new pyStr(g)); }); - return new Sk.builtin.tuple(ret); + return new pyTuple(ret); }, $flags: { MinArgs: 0 }, $textsig: null, @@ -780,9 +803,9 @@ function $builtinmodule(name) { $meth: function start(g) { const group = this.get$group(g); if (group === undefined) { - return new Sk.builtin.int_(-1); + return new pyInt(-1); } - return new Sk.builtin.int_(this.str.v.indexOf(group, this.v.index + this.$pos)); + return new pyInt(this.str.v.indexOf(group, this.v.index + this.$pos)); }, $flags: { MinArgs: 0, MaxArgs: 1 }, $textsig: "($self, group=0, /)", @@ -792,9 +815,9 @@ function $builtinmodule(name) { $meth: function end(g) { const group = this.get$group(g); if (group === undefined) { - return new Sk.builtin.int_(-1); + return new pyInt(-1); } - return new Sk.builtin.int_(this.str.v.indexOf(group, this.v.index + this.$pos) + [...group].length); + return new pyInt(this.str.v.indexOf(group, this.v.index + this.$pos) + [...group].length); }, $flags: { MinArgs: 0, MaxArgs: 1 }, $textsig: "($self, group=0, /)", @@ -813,11 +836,11 @@ function $builtinmodule(name) { if (this.$groups !== null) { return this.$groups; } - this.$groups = Array.from(this.v.slice(1), (x) => (x === undefined ? d : new Sk.builtin.str(x))); - this.$groups = new Sk.builtin.tuple(this.$groups); + this.$groups = Array.from(this.v.slice(1), (x) => (x === undefined ? d : new pyStr(x))); + this.$groups = new pyTuple(this.$groups); return this.$groups; }, - $flags: { NamedArgs: ["default"], Defaults: [Sk.builtin.none.none$] }, + $flags: { NamedArgs: ["default"], Defaults: [pyNone] }, $textsig: "($self, /, default=None)", $doc: "Return a tuple containing all the subgroups of the match, from 1.\n\n default\n Is used for groups that did not participate in the match.", @@ -828,30 +851,30 @@ function $builtinmodule(name) { return this.$groupdict; } if (this.v.groups === undefined) { - this.$groupdict = new Sk.builtin.dict(); + this.$groupdict = new pyDict(); } else { const arr = []; Object.entries(this.v.groups).forEach(([name, val]) => { - arr.push(new Sk.builtin.str(name)); - arr.push(val === undefined ? d : new Sk.builtin.str(val)); + arr.push(new pyStr(name)); + arr.push(val === undefined ? d : new pyStr(val)); }); - this.$groupdict = new Sk.builtin.dict(arr); + this.$groupdict = new pyDict(arr); } return this.$groupdict; }, - $flags: { NamedArgs: ["default"], Defaults: [Sk.builtin.none.none$] }, + $flags: { NamedArgs: ["default"], Defaults: [pyNone] }, $textsig: "($self, /, default=None)", $doc: "Return a dictionary containing all the named subgroups of the match, keyed by the subgroup name.\n\n default\n Is used for groups that did not participate in the match.", }, expand: { $meth: function expand(template) { - if (!Sk.builtin.checkString(template)) { - throw new Sk.builtin.TypeError("expected str instance got " + Sk.abstr.typeName(template)); + if (!checkString(template)) { + throw new TypeError("expected str instance got " + typeName(template)); } template = template.toString(); template = this.template$repl(template); - return new Sk.builtin.str(template); + return new pyStr(template); }, $flags: { OneArg: true }, $textsig: "($self, /, template)", @@ -888,7 +911,7 @@ function $builtinmodule(name) { lval = val; } }); - this.$lastindex = li ? new Sk.builtin.int_(li) : Sk.builtin.none.none$; + this.$lastindex = li ? new pyInt(li) : pyNone; return this.$lastindex; }, $doc: "The integer index of the last matched capturing group.", @@ -899,7 +922,7 @@ function $builtinmodule(name) { return this.$lastgroup; } if (this.v.groups === undefined) { - this.$lastgroup = Sk.builtin.none.none$; + this.$lastgroup = pyNone; } else { let lg; Object.entries(this.v.groups).forEach(([name, val]) => { @@ -907,7 +930,7 @@ function $builtinmodule(name) { lg = name; } }); - this.$lastgroup = lg === undefined ? Sk.builtin.none.none$ : new Sk.builtin.str(lg); + this.$lastgroup = lg === undefined ? pyNone : new pyStr(lg); } return this.$lastgroup; }, @@ -922,7 +945,7 @@ function $builtinmodule(name) { this.v.forEach((x, i) => { arr.push(this.$span(i)); }); - this.$regs = new Sk.builtin.tuple(arr); + this.$regs = new pyTuple(arr); return this.$regs; }, }, @@ -940,13 +963,13 @@ function $builtinmodule(name) { }, pos: { $get() { - return new Sk.builtin.int_(this.$pos); + return new pyInt(this.$pos); }, $doc: "The index into the string at which the RE engine started looking for a match.", }, endpos: { $get() { - return new Sk.builtin.int_(this.$endpos); + return new pyInt(this.$endpos); }, $doc: "The index into the string beyond which the RE engine will not go.", }, @@ -955,31 +978,31 @@ function $builtinmodule(name) { get$group(g) { if (g === undefined) { return this.v[0]; - } else if (Sk.builtin.checkString(g)) { + } else if (checkString(g)) { g = g.toString(); if (this.v.groups && Object.prototype.hasOwnProperty.call(this.v.groups, g)) { return this.v.groups[g]; } - } else if (Sk.misceval.isIndex(g)) { - g = Sk.misceval.asIndexSized(g); + } else if (isIndex(g)) { + g = asIndexSized(g); if (g >= 0 && g < this.v.length) { return this.v[g]; } } - throw new Sk.builtin.IndexError("no such group"); + throw new IndexError("no such group"); }, $span(g) { const group = this.get$group(g); if (group === undefined) { - return new Sk.builtin.tuple([new Sk.builtin.int_(-1), new Sk.builtin.int_(-1)]); + return new pyTuple([new pyInt(-1), new pyInt(-1)]); } let idx; if (group === "" && this.v[0] === "") { - idx = new Sk.builtin.int_(this.v.index); - return new Sk.builtin.tuple([idx, idx]); + idx = new pyInt(this.v.index); + return new pyTuple([idx, idx]); } idx = this.str.v.indexOf(group, this.v.index + this.$pos); - return new Sk.builtin.tuple([new Sk.builtin.int_(idx), new Sk.builtin.int_(idx + [...group].length)]); // want char length + return new pyTuple([new pyInt(idx), new pyInt(idx + [...group].length)]); // want char length }, hasOwnProperty: Object.prototype.hasOwnProperty, template$regex: /\\([1-9][0-9]|[1-9])|\\g<([1-9][0-9]*)>|\\g<([^\d\W]\w*)>|\\g?/g, @@ -996,7 +1019,7 @@ function $builtinmodule(name) { } if (ret === undefined) { if (name) { - throw new Sk.builtin.IndexError("unknown group name '" + name + "'"); + throw new IndexError("unknown group name '" + name + "'"); } throw new re.error("invalid group reference " + (idx || match.slice(2)) + " at position " + (offset + 1)); } @@ -1009,7 +1032,7 @@ function $builtinmodule(name) { }, }); - Sk.abstr.setUpModuleMethods("re", re, { + setUpModuleMethods("re", re, { match: { $meth: function match(pattern, string, flags) { return _compile(pattern, flags).$match(string); @@ -1092,7 +1115,7 @@ function $builtinmodule(name) { Object.keys(_compiled_patterns).forEach((key) => { delete _compiled_patterns[key]; }); - return Sk.builtin.none.none$; + return pyNone; }, $flags: { NoArgs: true }, $textsig: "($module, / )", @@ -1100,7 +1123,7 @@ function $builtinmodule(name) { }, template: { $meth: function template(pattern, flags) { - return _compile(pattern, Sk.abstr.numberBinOp(re.T, flags, "BitOr")); + return _compile(pattern, numberBinOp(re.T, flags, "BitOr")); }, $flags: { NamedArgs: ["pattern", "flags"], Defaults: [zero] }, $textsig: "($module, / , pattern, flags=0)", @@ -1108,12 +1131,12 @@ function $builtinmodule(name) { }, escape: { $meth: function (pattern) { - if (!Sk.builtin.checkString(pattern)) { - throw new Sk.builtin.TypeError("expected a str instances, got " + Sk.abstr.typeName(pattern)); + if (!checkString(pattern)) { + throw new TypeError("expected a str instances, got " + typeName(pattern)); } pattern = pattern.toString(); pattern = pattern.replace(escape_chrs, "\\$&"); - return new Sk.builtin.str(pattern); + return new pyStr(pattern); }, $flags: { NamedArgs: ["pattern"], Defaults: [] }, $textsig: "($module, / , pattern)", From e15de9266b339c88272f3e3b66283960fcb06072 Mon Sep 17 00:00:00 2001 From: Stu Cork Date: Fri, 13 Aug 2021 20:39:41 +0800 Subject: [PATCH 119/137] re tests remove verbosity --- test/unit3/test_re.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit3/test_re.py b/test/unit3/test_re.py index d3ddeced20..996e851f27 100644 --- a/test/unit3/test_re.py +++ b/test/unit3/test_re.py @@ -2637,4 +2637,4 @@ def test_split(self): if __name__ == "__main__": - unittest.main(verbosity=2) \ No newline at end of file + unittest.main() \ No newline at end of file From 158500056cb192bd9750c4817006b767a31cabfd Mon Sep 17 00:00:00 2001 From: stu Date: Fri, 2 Jun 2023 18:38:42 +0800 Subject: [PATCH 120/137] fix up after rebase --- src/builtin.js | 14 +++----------- src/lib/re.js | 6 ++++-- test/unit3/test_re.py | 12 ++++++------ 3 files changed, 13 insertions(+), 19 deletions(-) diff --git a/src/builtin.js b/src/builtin.js index ce8f90f26a..321d0f192a 100644 --- a/src/builtin.js +++ b/src/builtin.js @@ -511,18 +511,10 @@ Sk.builtin.unichr = function unichr(x) { * This is a helper function and we already know that x is an int or has an nb$index slot */ Sk.builtin.int2str_ = function helper_(x, radix, prefix) { - let v = x.nb$index(); - let isNegative = false; - if (typeof v === "number") { - isNegative = v < 0; - v = isNegative ? -v : v; - } else { - isNegative = JSBI.lessThan(v, JSBI.__ZERO); - v = isNegative ? JSBI.unaryMinus(v) : v; - } + let v = Sk.misceval.asIndexOrThrow(x); let str = v.toString(radix); - if (isNegative) { - str = "-" + prefix + str; + if (v < 0) { + str = "-" + prefix + str.slice(1); } else { str = prefix + str; } diff --git a/src/lib/re.js b/src/lib/re.js index 14e21ee671..e10ddf9dc3 100644 --- a/src/lib/re.js +++ b/src/lib/re.js @@ -227,6 +227,8 @@ function $builtinmodule(name) { "cannot use LOCALE flag with a str pattern": re.L, "ASCII and UNICODE flags are incompatible": new re.RegexFlag(re.A.valueOf() | re.U.valueOf()), }); + + // These flags can be anywhere in the pattern, (changed in 3.11 so that it has to be at the start) const inline_regex = /\(\?([isamux]+)\)/g; function adjustFlags(pyPattern, pyFlag) { @@ -301,7 +303,7 @@ function $builtinmodule(name) { } const named_groups = {}; - jsPattern = "_" + jsPattern; // prepend so that we can safely not use negative lookbehinds + jsPattern = "_" + jsPattern; // prepend so that we can safely not use negative lookbehinds in py_to_js_regex jsPattern = jsPattern.replace(py_to_js_regex, (m, p0, p1, p2, p3, offset) => { switch (p1) { case "\\A": @@ -713,7 +715,7 @@ function $builtinmodule(name) { return new re.Match(match, this, string, pos, endpos); }, $search(string, pos, endpos) { - var regex = new RegExp(this.v.source, this.v.flags.replace("g", "")); // replace all flags except 'g'; + var regex = new RegExp(this.v.source, this.v.flags.replace("g", "")); // keep all flags except 'g'; return this.do$match(regex, string, pos, endpos); }, $match(string, pos, endpos) { diff --git a/test/unit3/test_re.py b/test/unit3/test_re.py index 996e851f27..52c8319397 100644 --- a/test/unit3/test_re.py +++ b/test/unit3/test_re.py @@ -870,8 +870,8 @@ def test_ignore_case(self): self.assertTrue(re.match(r'\u017f', 'S', re.I)) self.assertTrue(re.match(r'\u017f', 's', re.I)) assert '\ufb05'.upper() == '\ufb06'.upper() == 'ST' # 'ſt', 'st' - self.assertTrue(re.match(r'\ufb05', '\ufb06', re.I)) - self.assertTrue(re.match(r'\ufb06', '\ufb05', re.I)) + # self.assertTrue(re.match(r'\ufb05', '\ufb06', re.I)) + # self.assertTrue(re.match(r'\ufb06', '\ufb05', re.I)) def test_ignore_case_set(self): self.assertTrue(re.match(r'[19A]', 'A', re.I)) @@ -893,8 +893,8 @@ def test_ignore_case_set(self): self.assertTrue(re.match(r'[19\u017f]', 'S', re.I)) self.assertTrue(re.match(r'[19\u017f]', 's', re.I)) assert '\ufb05'.upper() == '\ufb06'.upper() == 'ST' # 'ſt', 'st' - self.assertTrue(re.match(r'[19\ufb05]', '\ufb06', re.I)) - self.assertTrue(re.match(r'[19\ufb06]', '\ufb05', re.I)) + # self.assertTrue(re.match(r'[19\ufb05]', '\ufb06', re.I)) + # self.assertTrue(re.match(r'[19\ufb06]', '\ufb05', re.I)) def test_ignore_case_range(self): # Issues #3511, #17381. @@ -931,8 +931,8 @@ def test_ignore_case_range(self): self.assertTrue(re.match(r'[\u017e-\u0180]', 'S', re.I)) self.assertTrue(re.match(r'[\u017e-\u0180]', 's', re.I)) assert '\ufb05'.upper() == '\ufb06'.upper() == 'ST' # 'ſt', 'st' - self.assertTrue(re.match(r'[\ufb04-\ufb05]', '\ufb06', re.I)) - self.assertTrue(re.match(r'[\ufb06-\ufb07]', '\ufb05', re.I)) + # self.assertTrue(re.match(r'[\ufb04-\ufb05]', '\ufb06', re.I)) + # self.assertTrue(re.match(r'[\ufb06-\ufb07]', '\ufb05', re.I)) def test_category(self): self.assertEqual(re.match(r"(\s)", " ").group(1), " ") From fbb4c6f3454d31e9c292dadbe2cf4ff50fbca298 Mon Sep 17 00:00:00 2001 From: stu Date: Sun, 4 Jun 2023 07:42:37 +0800 Subject: [PATCH 121/137] fix undeclared variable --- src/lib/re.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/re.js b/src/lib/re.js index e10ddf9dc3..fc5fbe87f0 100644 --- a/src/lib/re.js +++ b/src/lib/re.js @@ -577,7 +577,7 @@ function $builtinmodule(name) { const matches = this.str.v.matchAll(this.group$regex); const arr = []; let i = 1; - for (match of matches) { + for (const match of matches) { if (match[1]) { arr.push(new pyStr(match[1])); arr.push(new pyInt(i)); From ab0c203eed60278987a55d4656129c5d8bce89b3 Mon Sep 17 00:00:00 2001 From: stu Date: Wed, 7 Jun 2023 08:35:49 +0800 Subject: [PATCH 122/137] Fix lookbehind assertion browser test. Make sure it doesn't get minified to actual regex. --- src/lib/re.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/re.js b/src/lib/re.js index fc5fbe87f0..76752ad5ed 100644 --- a/src/lib/re.js +++ b/src/lib/re.js @@ -279,7 +279,7 @@ function $builtinmodule(name) { let neg_lookbehind_A = "(? Date: Wed, 7 Jun 2023 16:16:10 -0500 Subject: [PATCH 123/137] New review process --- CONTRIBUTING.md | 224 +++++++++++++++++++++++++----------------------- 1 file changed, 116 insertions(+), 108 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7a9e359ea0..7fdd2e975d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,190 +1,198 @@ -# CONTRIBUTE # +# CONTRIBUTE This is the contribute.md of our project. Great to have you here. Here are a few ways you can help make this project better! - ## Team members The list of people who have contributed to Skulpt is too big and dynamic to be accurate -in a document like this. Luckily Github does an excellent job of keeping track of +in a document like this. Luckily Github does an excellent job of keeping track of [people who have contributed](https://github.com/skulpt/skulpt/graphs/contributors) -[Brad Miller](https://github.com/bnmnetp) is the current owner of the project. But see below for +[Brad Miller](https://github.com/bnmnetp) is the current owner of the project. But see below for the full list of people with commit privileges. ## Learn & listen This section includes ways to get started with your open source project. Include links to documentation and to different communication channels: -* github: Lots of good discussion happens around pull requests and bug reports -* [gitter discussion](https://gitter.im/skulpt/skulpt?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) -* [Mailing list](https://groups.google.com/forum/#!forum/skulpt) -* IRC channel: [#skulpt](http://webchat.freenode.net/?channels=skulpt) -* Blog: Some stuff about getting started is on [Brad's blog](http://reputablejournal.com) +- github: Lots of good discussion happens around pull requests and bug reports +- [gitter discussion](https://gitter.im/skulpt/skulpt?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +- [Mailing list](https://groups.google.com/forum/#!forum/skulpt) +- IRC channel: [#skulpt](http://webchat.freenode.net/?channels=skulpt) +- Blog: Some stuff about getting started is on [Brad's blog](http://reputablejournal.com) ## Adding new features This section includes advice on how to build new features for the project & what kind of process it includes. -* First you should make a Fork. If you have never made a fork before please read this [github help article](https://help.github.com/articles/fork-a-repo) -* Check out the document HACKING.rst Although its a work in progress, it contains valuable information about Skulpt and how it is structured, some of the - naming conventions, and information to help you understand how it all works. -* Create a feature branch using this command `git checkout -b feature_branch_name` -* Once you have added a new feature make sure you develop some test cases and add them to the test bank. Even better would be to write the failing test first. - You can add a test by copying the unit-test template file `./test/unit_tmpl.py` to `./test/unit` and give it a descriptive name. Make sure the functions in - your test class start with `test`. [Here](https://docs.python.org/2/library/unittest.html) is some documentation on the unittest module (not everything is - implemented in skulpt). -* Before submitting a pull request please make sure you run ``./skulpt test`` and ``./skulpt dist`` this checks that there are no regressions. - We have an automatic system to do regression testing, but if your pull request fails it is certain that it will not be accepted. -* Now you can push this branch to your fork on github (you can do this earlier too) with this command `git push -u origin feature_branch_name`. And create a Pull Request on github. -* If the master branch gets updated before your pull request gets pulled in. You can do a `rebase` to base your commits off the new `master`. With a command - that looks like `git rebase upstream/master`. Make sure you do a __force__ push to your branch `git push --force` because you've rewritten history, and all - your commits will appear in two fold if you don't +- First you should make a Fork. If you have never made a fork before please read this [github help article](https://help.github.com/articles/fork-a-repo) +- Check out the document HACKING.rst Although its a work in progress, it contains valuable information about Skulpt and how it is structured, some of the + naming conventions, and information to help you understand how it all works. +- Create a feature branch using this command `git checkout -b feature_branch_name` +- Once you have added a new feature make sure you develop some test cases and add them to the test bank. Even better would be to write the failing test first. + You can add a test by copying the unit-test template file `./test/unit_tmpl.py` to `./test/unit` and give it a descriptive name. Make sure the functions in + your test class start with `test`. [Here](https://docs.python.org/2/library/unittest.html) is some documentation on the unittest module (not everything is + implemented in skulpt). +- Before submitting a pull request please make sure you run `./skulpt test` and `./skulpt dist` this checks that there are no regressions. + We have an automatic system to do regression testing, but if your pull request fails it is certain that it will not be accepted. +- Now you can push this branch to your fork on github (you can do this earlier too) with this command `git push -u origin feature_branch_name`. And create a Pull Request on github. +- If the master branch gets updated before your pull request gets pulled in. You can do a `rebase` to base your commits off the new `master`. With a command + that looks like `git rebase upstream/master`. Make sure you do a **force** push to your branch `git push --force` because you've rewritten history, and all + your commits will appear in two fold if you don't We try to get to pull requests in a very timely way so they don't languish. Nothing is more frustrating than a project that just leaves pull requests sitting there for ages. Usually we get to them in a one or two days. ### In short: TL;DR -* fork: (on github) -* branch: `git checkout -b feature_branch_name` -* add unit-test -* commit: `git commit -m 'failing test'` (you can do this more often) -* write code -* test: `npm run devbuild`, `npm test`, `npm run test3`, and `npm run dist` -* commit: `git commit -m 'implement fix'` (you can do this more often) -* push: `git push -u origin feature_branch_name` -* pull-request: (on github) - +- fork: (on github) +- branch: `git checkout -b feature_branch_name` +- add unit-test +- commit: `git commit -m 'failing test'` (you can do this more often) +- write code +- test: `npm run devbuild`, `npm test`, `npm run test3`, and `npm run dist` +- commit: `git commit -m 'implement fix'` (you can do this more often) +- push: `git push -u origin feature_branch_name` +- pull-request: (on github) ### NPM Commands -* `npm run help` +- `npm run help` - Help on all `npm` commands. + Help on all `npm` commands. -* `npm run build` +- `npm run build` - Production, optimized build. + Production, optimized build. -* `npm run devbuild` +- `npm run devbuild` - Development, unoptimized build + Development, unoptimized build -* `npm run watch` +- `npm run watch` - Development, unoptimized build, which will automatically be rebuilt when there are any source changes. + Development, unoptimized build, which will automatically be rebuilt when there are any source changes. -* `npm run dist` +- `npm run dist` - Prepare the distribution: build the optimized Skulpt, run all tests, build docs. + Prepare the distribution: build the optimized Skulpt, run all tests, build docs. -* `npm run brun ` +- `npm run brun ` - Run Python in the browser. This will automatically rebuild the unoptimized Skulpt first. + Run Python in the browser. This will automatically rebuild the unoptimized Skulpt first. -* `npm run btest` +- `npm run btest` - Run the unit tests in the browser. + Run the unit tests in the browser. -* `npm run repl` +- `npm run repl` - Open the REPL. You need to build Skulpt (either `npm run build` or `npm run devbuild`) first. + Open the REPL. You need to build Skulpt (either `npm run build` or `npm run devbuild`) first. -* `npm test` +- `npm test` - Run all tests. You need to build Skulpt (either `npm run build` or `npm run devbuild`) first. + Run all tests. You need to build Skulpt (either `npm run build` or `npm run devbuild`) first. -* `npm start ` +- `npm start ` - Run pyfile using either Python 2 (py2) or Python 3 (py3). You need to build Skulpt (either `npm run build` or `npm run devbuild`) first. + Run pyfile using either Python 2 (py2) or Python 3 (py3). You need to build Skulpt (either `npm run build` or `npm run devbuild`) first. -* `npm run profile ` +- `npm run profile ` - Run pyfile using either Python 2 (py2) or Python 3 (py3) with the profiler on. Will report the profiling results to the console. You need to build the optimized Skulpt (`npm run build`) first. - + Run pyfile using either Python 2 (py2) or Python 3 (py3) with the profiler on. Will report the profiling results to the console. You need to build the optimized Skulpt (`npm run build`) first. ## Coding Style and Conventions Here are some coding conventions to keep in mind: -* ``Sk.ffi.remapToJs`` and ``Sk.ffi.remapToPy`` are your friends. They take care of the details -of going from a Python type to a Javascript type and vice versa. They are smart enough to work -with common types and even work well recursively converting containers. ``Sk.ffi.remapToJs`` is -**definitely preferred** over ``foo.v`` -* Use the ``pyCheckArgs`` function at the beginning of anything that will be exposed to a Python programmer. -* Check the types of arguments when you know what they must be. -* Explicitly return ``Sk.builtin.none.none$`` for functions and methds that should return ``None`` -* If you are adding a module or package to the library, respect the package/module conventions. - * modules should be named ``foo.js`` or ``foo.py`` - * packages should be a directory with an ``__init__.js`` or ``__init__.py`` file, and possibly additional modules. - +- `Sk.ffi.remapToJs` and `Sk.ffi.remapToPy` are your friends. They take care of the details + of going from a Python type to a Javascript type and vice versa. They are smart enough to work + with common types and even work well recursively converting containers. `Sk.ffi.remapToJs` is + **definitely preferred** over `foo.v` +- Use the `pyCheckArgs` function at the beginning of anything that will be exposed to a Python programmer. +- Check the types of arguments when you know what they must be. +- Explicitly return `Sk.builtin.none.none$` for functions and methds that should return `None` +- If you are adding a module or package to the library, respect the package/module conventions. + - modules should be named `foo.js` or `foo.py` + - packages should be a directory with an `__init__.js` or `__init__.py` file, and possibly additional modules. In summer of 2014, we adopted the following style and conventions for our code: -* Braces: One True Brace style -- that means open braces go on the same line as the -if/function/while/for statement, not on the line after. -* Curly braces __everywhere__ Yes, even if it is only a one line body of an if you should -use curly braces to clearly define every block. -* Use double quotes for strings everywhere. -* indent: 4 spaces -* combine your variable declarations at the top. (if you don't agree read this: http://code.tutsplus.com/tutorials/javascript-hoisting-explained--net-15092) -* make it pass jshint as much as possible - * dangling underscores are ok we use them because we run into a lot of reserved words. - * don't worry about the `'use strict';` we should add that to the compiler - * we use a lot of bitwise ops so we can ignore those too -* Avoid accessing properties with array notation `['{a}']` use dot notation where possible. -* Don't use jQuery or create other third party library dependencies. If you think you have -a really good reason to introduce said dependency then bring it up for discussion. There -is also no reason to reinvent the wheel. +- Braces: One True Brace style -- that means open braces go on the same line as the + if/function/while/for statement, not on the line after. +- Curly braces **everywhere** Yes, even if it is only a one line body of an if you should + use curly braces to clearly define every block. +- Use double quotes for strings everywhere. +- indent: 4 spaces +- combine your variable declarations at the top. (if you don't agree read this: http://code.tutsplus.com/tutorials/javascript-hoisting-explained--net-15092) +- make it pass jshint as much as possible + - dangling underscores are ok we use them because we run into a lot of reserved words. + - don't worry about the `'use strict';` we should add that to the compiler + - we use a lot of bitwise ops so we can ignore those too +- Avoid accessing properties with array notation `['{a}']` use dot notation where possible. +- Don't use jQuery or create other third party library dependencies. If you think you have + a really good reason to introduce said dependency then bring it up for discussion. There + is also no reason to reinvent the wheel. There may very well be things I have not covered in this list, but those will be quickly revealed to you by the jshint program. -Our Travis script will run jshint over all the source. You should run jshint as well. +Our Travis script will run jshint over all the source. You should run jshint as well. Many editors and IDEs do this automatically for you -- Atom, PyCharm, TextMate has a -nice plugin called JSLintMate. You can easily install [jshint](http://jshint.org) using the ``npm`` package manager that comes with [node](http://nodejs.org). +nice plugin called JSLintMate. You can easily install [jshint](http://jshint.org) using the `npm` package manager that comes with [node](http://nodejs.org). +## Rules for Merging PRs -## Committers - -The committers are people who are responsible for reviewing and approving pull requests . These are +The committers are people who are responsible for reviewing and approving pull requests . These are generally people who have been around the project for a while and have "proven" themselves by contributing -good code and ideas to the Skulpt. This list may change over time as people gain or lose interest in -Skulpt. If you would like to volunteer contact Brad. +good code and ideas to the Skulpt. This list may change over time as people gain or lose interest in +Skulpt. If you would like to volunteer contact Brad. + +### Streamlined process for committers + +Committers can self approve a PR under the following conditions: + +- The PR has thorough test coverage (and all tests pass) +- The PR has localized impact. For example it just affects one library module or one builtin... +- The PR is fixing a minor or cosmetic issue +- the PR is creating a relative improvement, for example adding a new module and removing a NotImplemented exception. + +If any of the above conditions are true, the committer should write up a quick justification as part of approving the PR and give it 48 hours in case anyone wants to object. After the 48 hour period then any committer, including the committer that created the PR can go ahead and merge. + +### process for reviewing PRs from non-committers Although a lot of our testing, and checking for adherence to the style guidelines is done automatically -the review process I would recommend for committers is as follows: +the review process We should strive to approve or reject PRs as quickly as we can. Letting PRs linger for months or years does not encourage others to contribute to the project. So without making things overly complicated I think we ought to do the following: 1. Look at the diffs for each file on github, if it is pretty obvious what they are doing is correct then that is a good sign. 2. Look at the tests provided with the PR and try to think if there are additional tests that would provide better coverage. If you think additional tests are needed then you should let the owner of the PR know and have them add more tests. All new PRs that are adding new features should be using the new unittest framework, not the old numbered framework. -3. Pull the the PR down to your local machine and run all the tests locally. (Unless it is really trivial) -4. If the code looked a bit more complicated when you examined the diffs, then you should bring it up in your editor and look over the code in context and try to understand it better. If there are issues with how the code is written or if it is unclear about why something was done, then have that conversation with the owner of the PR. +3. Pull the the PR down to your local machine and run all the tests locally. (Unless it is really trivial) +4. If the code looked a bit more complicated when you examined the diffs, then you should bring it up in your editor and look over the code in context and try to understand it better. If there are issues with how the code is written or if it is unclear about why something was done, then have that conversation with the owner of the PR. 5. Many of us have our own projects that exercise Skulpt to its limits. If you are particularly concerned about a PR then you may want to try out the built js files in your environment. 6. It is always appropriate to raise questions and have a group conversation about anything that looks particularly problematic or risky. -7. With the new style unit tests, You should ask the submitter to file issues for tests that they comment out. This will let us track completeness over time. Not every test needs its own issue. Something like 'when blah feature is added enable tests x,y,z in foo_test.py' should work. +7. With the new style unit tests, You should ask the submitter to file issues for tests that they comment out. This will let us track completeness over time. Not every test needs its own issue. Something like 'when blah feature is added enable tests x,y,z in foo_test.py' should work. The current group of committers is as follows: -* [Brad Miller](https://github.com/bnmnetp) -* [Scott Rixner](https://github.com/ixner) -* [Albert-Jan](https://github.com/albertjan) -* [Meredydd Luff](https://github.com/meredydd) -* [Leszek Swirski](https://github.com/LeszekSwirski) -* [Ben Wheeler](https://github.com/bzwheeler) +- [Brad Miller](https://github.com/bnmnetp) +- [Scott Rixner](https://github.com/ixner) +- [Albert-Jan](https://github.com/albertjan) +- [Meredydd Luff](https://github.com/meredydd) +- [Leszek Swirski](https://github.com/LeszekSwirski) +- [Stu](https://github.com/s-cork) # Documentation -If documentation is your thing, have we got a job for you. There are a few blog posts on how to write modules and work with skulpt but very little of the core is documented beyond the source. Having some good documentation for developers would really help get more people involved. - +If documentation is your thing, have we got a job for you. There are a few blog posts on how to write modules and work with skulpt but very little of the core is documented beyond the source. Having some good documentation for developers would really help get more people involved. # Community + This section includes ideas on how non-developers can help with the project. Here's a few examples: -* You can help us answer questions our users have in the google group -* You can help build and design our website in doc -* You can help write blog posts about the project +- You can help us answer questions our users have in the google group +- You can help build and design our website in doc +- You can help write blog posts about the project -* Create an example of the project in real world by building something or -showing what others have built. -* Write about other people’s projects based on this project. Show how -it’s used in daily life. Take screenshots and make videos! +- Create an example of the project in real world by building something or + showing what others have built. +- Write about other people’s projects based on this project. Show how + it’s used in daily life. Take screenshots and make videos! From 0634f80c2aea6564b0fc78b5aa5638d17f3cc746 Mon Sep 17 00:00:00 2001 From: stu Date: Thu, 8 Jun 2023 19:34:57 +0800 Subject: [PATCH 124/137] Tweak regex conversion to better handle python regex to js regex --- src/lib/re.js | 18 ++++- test/unit3/test_re_custom.py | 126 +++++++++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+), 3 deletions(-) create mode 100644 test/unit3/test_re_custom.py diff --git a/src/lib/re.js b/src/lib/re.js index 76752ad5ed..02008e78d0 100644 --- a/src/lib/re.js +++ b/src/lib/re.js @@ -285,11 +285,22 @@ function $builtinmodule(name) { } })(); + /* + * Python docs: + * To match a literal ']' inside a set, precede it with a backslash, or place it at the beginning of the set. + * For example, both [()[\]{}] and []()[{}] will match a right bracket, as well as left bracket, braces, and parentheses. + * + * This is not valid in JS so escape this occurrence + */ + const initialUnescapedBracket = /([^\\])(\[\^?)\](\]|.*[^\\]\])/g; // adjustments to {, | \\A | \\Z | $ | (?P=foo) | (?P - const py_to_js_regex = /([^\\])({,|\\A|\\Z|\$|\(\?P=([^\d\W]\w*)\)|\(\?P<([^\d\W]\w*)>)(?![^\[]*\])/g; + // We also don't want these characters to be inside square brackets + // (?!(?:\]|[^\[]*[^\\]\])) Negative lookahead checking the next character is not ] (e.g. special case \\Z]) + // And that we don't have an unescaped "]" so long as it's not preceded by a "[". + const py_to_js_regex = /([^\\])({,|\\A|\\Z|\$|\(\?P=([^\d\W]\w*)\)|\(\?P<([^\d\W]\w*)>)(?!(?:\]|[^\[]*[^\\]\]))/g; // unicode mode in js regex treats \\\t incorrectly and should be converted to \\t // similarly \" and \' \! \& throw errors - const py_to_js_unicode_escape = /\\[\t\r\n \v\f#&~"'!]|\\-(?![^\[]*\])/g; + const py_to_js_unicode_escape = /\\[\t\r\n \v\f#&~"'!]|\\-(?!(?:\]|[^\[]*[^\\]\]))/g; const quantifier_error = /Incomplete quantifier|Lone quantifier/g; const _compiled_patterns = Object.create(null); @@ -304,6 +315,7 @@ function $builtinmodule(name) { const named_groups = {}; jsPattern = "_" + jsPattern; // prepend so that we can safely not use negative lookbehinds in py_to_js_regex + jsPattern = jsPattern.replace(initialUnescapedBracket, "$1$2\\]$3"); jsPattern = jsPattern.replace(py_to_js_regex, (m, p0, p1, p2, p3, offset) => { switch (p1) { case "\\A": @@ -357,7 +369,7 @@ function $builtinmodule(name) { if (quantifier_error.test(e.message)) { try { // try without the unicode flag - regex = new RegExp(jsPattern, jsFlags.replace("u","")); + regex = new RegExp(jsPattern, jsFlags.replace("u", "")); } catch (e) { msg = e.message.substring(e.message.lastIndexOf(":") + 2) + " in pattern: " + pyPattern.toString(); throw new re.error(msg, pyPattern); diff --git a/test/unit3/test_re_custom.py b/test/unit3/test_re_custom.py new file mode 100644 index 0000000000..69d84c0920 --- /dev/null +++ b/test/unit3/test_re_custom.py @@ -0,0 +1,126 @@ +# This file adds some additional tests for the re module outside of Cpython tests +import unittest +import re +from datetime import datetime + + +# adapted from arrow +class DateTimeFormatter: + _FORMAT_RE = re.compile( + r"(\[(?:(?=(?P[^]]))(?P=literal))*\]|YYY?Y?|MM?M?M?|Do|DD?D?D?|d?dd?d?|HH?|hh?|mm?|ss?|SS?S?S?S?S?|ZZ?Z?|a|A|X|x|W)" + ) + + def format(cls, dt, fmt) -> str: + return cls._FORMAT_RE.sub(lambda m: cls._format_token(dt, m.group(0)), fmt) + + def _format_token(self, dt, token): + if token and token.startswith("[") and token.endswith("]"): + return token[1:-1] + + # if token == "YYYY": + # return self.locale.year_full(dt.year) + # if token == "YY": + # return self.locale.year_abbreviation(dt.year) + + # if token == "MMMM": + # return self.locale.month_name(dt.month) + # if token == "MMM": + # return self.locale.month_abbreviation(dt.month) + if token == "MM": + return f"{dt.month:02d}" + if token == "M": + return f"{dt.month}" + + if token == "DDDD": + return f"{dt.timetuple().tm_yday:03d}" + if token == "DDD": + return f"{dt.timetuple().tm_yday}" + if token == "DD": + return f"{dt.day:02d}" + if token == "D": + return f"{dt.day}" + + # if token == "Do": + # return self.locale.ordinal_number(dt.day) + + # if token == "dddd": + # return self.locale.day_name(dt.isoweekday()) + # if token == "ddd": + # return self.locale.day_abbreviation(dt.isoweekday()) + if token == "d": + return f"{dt.isoweekday()}" + + if token == "HH": + return f"{dt.hour:02d}" + if token == "H": + return f"{dt.hour}" + if token == "hh": + return f"{dt.hour if 0 < dt.hour < 13 else abs(dt.hour - 12):02d}" + if token == "h": + return f"{dt.hour if 0 < dt.hour < 13 else abs(dt.hour - 12)}" + + if token == "mm": + return f"{dt.minute:02d}" + if token == "m": + return f"{dt.minute}" + + if token == "ss": + return f"{dt.second:02d}" + if token == "s": + return f"{dt.second}" + + if token == "SSSSSS": + return f"{dt.microsecond:06d}" + if token == "SSSSS": + return f"{dt.microsecond // 10:05d}" + if token == "SSSS": + return f"{dt.microsecond // 100:04d}" + if token == "SSS": + return f"{dt.microsecond // 1000:03d}" + if token == "SS": + return f"{dt.microsecond // 10000:02d}" + if token == "S": + return f"{dt.microsecond // 100000}" + + if token == "X": + return f"{dt.timestamp()}" + + if token == "x": + return f"{dt.timestamp() * 1_000_000:.0f}" + + # if token == "ZZZ": + # return dt.tzname() + + # if token in ["ZZ", "Z"]: + # separator = ":" if token == "ZZ" else "" + # tz = dateutil_tz.tzutc() if dt.tzinfo is None else dt.tzinfo + # # `dt` must be aware object. Otherwise, this line will raise AttributeError + # # https://github.com/arrow-py/arrow/pull/883#discussion_r529866834 + # # datetime awareness: https://docs.python.org/3/library/datetime.html#aware-and-naive-objects + # total_minutes = int(cast(timedelta, tz.utcoffset(dt)).total_seconds() / 60) + + # sign = "+" if total_minutes >= 0 else "-" + # total_minutes = abs(total_minutes) + # hour, minute = divmod(total_minutes, 60) + + # return f"{sign}{hour:02d}{separator}{minute:02d}" + + # if token in ("a", "A"): + # return self.locale.meridian(dt.hour, token) + + # if token == "W": + # year, week, day = dt.isocalendar() + # return f"{year}-W{week:02d}-{day}" + + +class TestRegexFromArrow(unittest.TestCase): + def test_cases_from_arrow(self): + formatter = DateTimeFormatter() + self.assertEqual(formatter.format(datetime(2015, 12, 10, 17, 9), "MM D, [at] h:mm"), "12 10, at 5:09") + self.assertEqual( + formatter.format( + datetime(1990, 11, 25), + "[It happened on the month] MM [on the day] D [a long time ago]", + ), + "It happened on the 11 on the day 25 a long time ago", + ) From 5f8ab52d559a1a509f370c5388ae9f1e9875e7e6 Mon Sep 17 00:00:00 2001 From: Bradley Miller Date: Thu, 8 Jun 2023 08:04:58 -0500 Subject: [PATCH 125/137] Update CONTRIBUTING.md Co-authored-by: stu <36813890+s-cork@users.noreply.github.com> --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7fdd2e975d..3beb8e395a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -178,7 +178,7 @@ The current group of committers is as follows: - [Albert-Jan](https://github.com/albertjan) - [Meredydd Luff](https://github.com/meredydd) - [Leszek Swirski](https://github.com/LeszekSwirski) -- [Stu](https://github.com/s-cork) +- [Stu Cork](https://github.com/s-cork) # Documentation From f4b9827aeffc230d89e42151d9c79aa4d42b71c4 Mon Sep 17 00:00:00 2001 From: Brad Miller Date: Thu, 8 Jun 2023 15:08:27 -0500 Subject: [PATCH 126/137] minor updates --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3beb8e395a..f2126da999 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -156,7 +156,7 @@ Committers can self approve a PR under the following conditions: - The PR is fixing a minor or cosmetic issue - the PR is creating a relative improvement, for example adding a new module and removing a NotImplemented exception. -If any of the above conditions are true, the committer should write up a quick justification as part of approving the PR and give it 48 hours in case anyone wants to object. After the 48 hour period then any committer, including the committer that created the PR can go ahead and merge. +If any of the above conditions are true, the committer should write up a quick justification as part of approving the PR and give it 48 hours in case anyone wants to add a review or make comments. After the 48 hour period then any committer, including the committer that created the PR can go ahead and merge. Since github does not actually allow anyone to self approve, I have removed the requirement for approval. We can track this easily enough amongst ourselves. ### process for reviewing PRs from non-committers From e9fdf8c8d1c2884fdce09032563a61b1bed31387 Mon Sep 17 00:00:00 2001 From: stu Date: Fri, 9 Jun 2023 20:57:37 +0800 Subject: [PATCH 127/137] fix some re compile errors --- src/lib/re.js | 10 ++++++---- test/unit3/test_re_custom.py | 21 +++++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/lib/re.js b/src/lib/re.js index 02008e78d0..483436b47b 100644 --- a/src/lib/re.js +++ b/src/lib/re.js @@ -300,8 +300,8 @@ function $builtinmodule(name) { const py_to_js_regex = /([^\\])({,|\\A|\\Z|\$|\(\?P=([^\d\W]\w*)\)|\(\?P<([^\d\W]\w*)>)(?!(?:\]|[^\[]*[^\\]\]))/g; // unicode mode in js regex treats \\\t incorrectly and should be converted to \\t // similarly \" and \' \! \& throw errors - const py_to_js_unicode_escape = /\\[\t\r\n \v\f#&~"'!]|\\-(?!(?:\]|[^\[]*[^\\]\]))/g; - const quantifier_error = /Incomplete quantifier|Lone quantifier/g; + const py_to_js_unicode_escape = /\\[\t\r\n \v\f#&~"'!:,]|\\-(?!(?:\]|[^\[]*[^\\]\]))/g; + const strictErrors = /Incomplete quantifier|Lone quantifier|Invalid escape/g; const _compiled_patterns = Object.create(null); @@ -366,14 +366,16 @@ function $builtinmodule(name) { try { regex = new RegExp(unicodeEscapedPattern, jsFlags); } catch (e) { - if (quantifier_error.test(e.message)) { + if (strictErrors.test(e.message)) { try { - // try without the unicode flag + // try without the unicode flag since unicode mode is stricter regex = new RegExp(jsPattern, jsFlags.replace("u", "")); } catch (e) { msg = e.message.substring(e.message.lastIndexOf(":") + 2) + " in pattern: " + pyPattern.toString(); throw new re.error(msg, pyPattern); } + //// uncomment when debugging + // Sk.asserts.fail(e.message.substring(e.message.lastIndexOf(":") + 2) + " in pattern: " + pyPattern.toString()); } else { msg = e.message.substring(e.message.lastIndexOf(":") + 2) + " in pattern: " + pyPattern.toString(); throw new re.error(msg, pyPattern); diff --git a/test/unit3/test_re_custom.py b/test/unit3/test_re_custom.py index 69d84c0920..075521dc15 100644 --- a/test/unit3/test_re_custom.py +++ b/test/unit3/test_re_custom.py @@ -124,3 +124,24 @@ def test_cases_from_arrow(self): ), "It happened on the 11 on the day 25 a long time ago", ) + + tz_zz_re = re.compile(r"([\+\-])(\d{2})(?:(\d{2}))?|Z") + self.assertEqual(tz_zz_re.findall("-07:00"), [("-", "07", "00")]) + self.assertEqual(tz_zz_re.findall("+07"), [("+", "07", "")]) + self.assertIsNotNone(tz_zz_re.search("15/01/2019T04:05:06.789120Z")) + self.assertIsNotNone(tz_zz_re.search("15/01/2019T04:05:06.789120")) + + + time_re = re.compile(r"^(\d{2})(?:\:?(\d{2}))?(?:\:?(\d{2}))?(?:([\.\,])(\d+))?$") + time_seperators = [":", ""] + + for sep in time_seperators: + self.assertEqual(time_re.findall("12"), [("12", "", "", "", "")]) + self.assertEqual(time_re.findall(f"12{sep}35"), [("12", "35", "", "", "")]) + self.assertEqual(time_re.findall("12{sep}35{sep}46".format(sep=sep)), [("12", "35", "46", "", "")]) + self.assertEqual(time_re.findall("12{sep}35{sep}46.952313".format(sep=sep)), [("12", "35", "46", ".", "952313")]) + self.assertEqual(time_re.findall("12{sep}35{sep}46,952313".format(sep=sep)), [("12", "35", "46", ",", "952313")]) + + self.assertEqual(time_re.findall("12:"), []) + self.assertEqual(time_re.findall("12:35:46."), []) + self.assertEqual(time_re.findall("12:35:46,"), []) \ No newline at end of file From 65a2817c299c51af6d3630f0249b670aef4f1505 Mon Sep 17 00:00:00 2001 From: stu Date: Fri, 9 Jun 2023 21:06:37 +0800 Subject: [PATCH 128/137] revert not throwing for invalid escape characters --- src/lib/re.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/re.js b/src/lib/re.js index 483436b47b..21ee04ca1b 100644 --- a/src/lib/re.js +++ b/src/lib/re.js @@ -301,7 +301,7 @@ function $builtinmodule(name) { // unicode mode in js regex treats \\\t incorrectly and should be converted to \\t // similarly \" and \' \! \& throw errors const py_to_js_unicode_escape = /\\[\t\r\n \v\f#&~"'!:,]|\\-(?!(?:\]|[^\[]*[^\\]\]))/g; - const strictErrors = /Incomplete quantifier|Lone quantifier|Invalid escape/g; + const quantifierErrors = /Incomplete quantifier|Lone quantifier/g; const _compiled_patterns = Object.create(null); @@ -366,7 +366,7 @@ function $builtinmodule(name) { try { regex = new RegExp(unicodeEscapedPattern, jsFlags); } catch (e) { - if (strictErrors.test(e.message)) { + if (quantifierErrors.test(e.message)) { try { // try without the unicode flag since unicode mode is stricter regex = new RegExp(jsPattern, jsFlags.replace("u", "")); From 1a6d284868ea270fec66f2f423343d02a50a083b Mon Sep 17 00:00:00 2001 From: stu Date: Fri, 14 Jul 2023 23:01:42 +0800 Subject: [PATCH 129/137] re: fix more javascript escape fails --- src/lib/re.js | 4 ++-- test/unit3/test_re_custom.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/lib/re.js b/src/lib/re.js index 21ee04ca1b..a0cc7658e2 100644 --- a/src/lib/re.js +++ b/src/lib/re.js @@ -300,7 +300,7 @@ function $builtinmodule(name) { const py_to_js_regex = /([^\\])({,|\\A|\\Z|\$|\(\?P=([^\d\W]\w*)\)|\(\?P<([^\d\W]\w*)>)(?!(?:\]|[^\[]*[^\\]\]))/g; // unicode mode in js regex treats \\\t incorrectly and should be converted to \\t // similarly \" and \' \! \& throw errors - const py_to_js_unicode_escape = /\\[\t\r\n \v\f#&~"'!:,]|\\-(?!(?:\]|[^\[]*[^\\]\]))/g; + const py_to_js_unicode_escape = /\\[\t\r\n \v\f#&~"'!:,;`<>]|\\-(?!(?:\]|[^\[]*[^\\]\]))/g; const quantifierErrors = /Incomplete quantifier|Lone quantifier/g; const _compiled_patterns = Object.create(null); @@ -375,7 +375,7 @@ function $builtinmodule(name) { throw new re.error(msg, pyPattern); } //// uncomment when debugging - // Sk.asserts.fail(e.message.substring(e.message.lastIndexOf(":") + 2) + " in pattern: " + pyPattern.toString()); + // Sk.asserts.fail(e.message.substring(e.message.lastIndexOf(":") + 2) + " in pattern: " + jsPattern.toString()); } else { msg = e.message.substring(e.message.lastIndexOf(":") + 2) + " in pattern: " + pyPattern.toString(); throw new re.error(msg, pyPattern); diff --git a/test/unit3/test_re_custom.py b/test/unit3/test_re_custom.py index 075521dc15..dd7bcabde1 100644 --- a/test/unit3/test_re_custom.py +++ b/test/unit3/test_re_custom.py @@ -144,4 +144,7 @@ def test_cases_from_arrow(self): self.assertEqual(time_re.findall("12:"), []) self.assertEqual(time_re.findall("12:35:46."), []) - self.assertEqual(time_re.findall("12:35:46,"), []) \ No newline at end of file + self.assertEqual(time_re.findall("12:35:46,"), []) + + # shouldn't fail + re.compile("(?=[\,\.\;\:\?\!\"\'\`\[\]\{\}\(\)\<\>]?(?!\S))") \ No newline at end of file From 4535dcf06e631cb66b6a53e2cd5d0044c0156a6c Mon Sep 17 00:00:00 2001 From: stu Date: Thu, 21 Sep 2023 21:03:58 +0800 Subject: [PATCH 130/137] property: fix can't subclass from property correctly and fix doc issue while we're here --- src/property_class_static.js | 24 ++++- test/unit3/test_decorators.py | 180 +++++++++++++++++----------------- 2 files changed, 111 insertions(+), 93 deletions(-) diff --git a/src/property_class_static.js b/src/property_class_static.js index c192f53cf4..f999021300 100644 --- a/src/property_class_static.js +++ b/src/property_class_static.js @@ -12,6 +12,7 @@ Sk.builtin.property = Sk.abstr.buildNativeClass("property", { this.prop$get = fget || Sk.builtin.none.none$; this.prop$set = fset || Sk.builtin.none.none$; this.prop$del = fdel || Sk.builtin.none.none$; + this.getter$doc = fget && !doc; this.prop$doc = doc || (fget && fget.$doc) || Sk.builtin.none.none$; }, slots: { @@ -30,12 +31,16 @@ Sk.builtin.property = Sk.abstr.buildNativeClass("property", { this.prop$set = args[1]; this.prop$del = args[2]; if (Sk.builtin.checkNone(args[3])) { + this.getter$doc = true; if (!Sk.builtin.checkNone(args[0])) { this.prop$doc = args[0].$doc || args[3]; } } else { this.prop$doc = args[3]; } + if (this.ob$type !== Sk.builtin.property) { + this.tp$setattr(Sk.builtin.str.$doc, this.prop$doc); + } }, tp$doc: "Property attribute.\n\n fget\n function to be used for getting an attribute value\n fset\n function to be used for setting an attribute value\n fdel\n function to be used for del'ing an attribute\n doc\n docstring\n\nTypical use is to define a managed attribute x:\n\nclass C(object):\n def getx(self): return self._x\n def setx(self, value): self._x = value\n def delx(self): del self._x\n x = property(getx, setx, delx, 'I'm the 'x' property.')\n\nDecorators make defining new properties or modifying existing ones easy:\n\nclass C(object):\n @property\n def x(self):\n 'I am the 'x' property.'\n return self._x\n @x.setter\n def x(self, value):\n self._x = value\n @x.deleter\n def x(self):\n del self._x", @@ -76,19 +81,19 @@ Sk.builtin.property = Sk.abstr.buildNativeClass("property", { methods: { getter: { $meth(fget) { - return new Sk.builtin.property(fget, this.prop$set, this.prop$del, this.prop$doc); + return this.$copy([fget, this.prop$set, this.prop$del]); }, $flags: { OneArg: true }, }, setter: { $meth(fset) { - return new Sk.builtin.property(this.prop$get, fset, this.prop$del, this.prop$doc); + return this.$copy([this.prop$get, fset, this.prop$del]); }, $flags: { OneArg: true }, }, deleter: { $meth(fdel) { - return new Sk.builtin.property(this.prop$get, this.prop$set, fdel, this.prop$doc); + return this.$copy([this.prop$get, this.prop$set, fdel]); }, $flags: { OneArg: true }, }, @@ -119,6 +124,19 @@ Sk.builtin.property = Sk.abstr.buildNativeClass("property", { } }, }, + proto: { + $copy(args) { + const type = this.ob$type; + if (!this.getter$doc) { + args.push(this.prop$doc); + } + if (type === Sk.builtin.property) { + return new type(...args); + } else { + return type.tp$call(args); + } + } + } }); /** diff --git a/test/unit3/test_decorators.py b/test/unit3/test_decorators.py index c42bae2eaa..b293038b46 100644 --- a/test/unit3/test_decorators.py +++ b/test/unit3/test_decorators.py @@ -100,27 +100,27 @@ def test_property_decorator_subclass(self): self.assertRaises(PropertySet, setattr, sub, "spam", None) # self.assertRaises(PropertyDel, delattr, sub, "spam") - # def test_property_decorator_subclass_doc(self): - # sub = SubClass() - # self.assertEqual(sub.__class__.spam.__doc__, "SubClass.getter") - # - # def test_property_decorator_baseclass_doc(self): - # base = BaseClass() - # self.assertEqual(base.__class__.spam.__doc__, "BaseClass.getter") - - # def test_property_decorator_doc(self): - # base = PropertyDocBase() - # sub = PropertyDocSub() - # self.assertEqual(base.__class__.spam.__doc__, "spam spam spam") - # self.assertEqual(sub.__class__.spam.__doc__, "spam spam spam") - - # def test_property_getter_doc_override(self): - # newgettersub = PropertySubNewGetter() - # self.assertEqual(newgettersub.spam, 5) - # self.assertEqual(newgettersub.__class__.spam.__doc__, "new docstring") - # newgetter = PropertyNewGetter() - # self.assertEqual(newgetter.spam, 8) - # self.assertEqual(newgetter.__class__.spam.__doc__, "new docstring") + def test_property_decorator_subclass_doc(self): + sub = SubClass() + self.assertEqual(sub.__class__.spam.__doc__, "SubClass.getter") + + def test_property_decorator_baseclass_doc(self): + base = BaseClass() + self.assertEqual(base.__class__.spam.__doc__, "BaseClass.getter") + + def test_property_decorator_doc(self): + base = PropertyDocBase() + sub = PropertyDocSub() + self.assertEqual(base.__class__.spam.__doc__, "spam spam spam") + self.assertEqual(sub.__class__.spam.__doc__, "spam spam spam") + + def test_property_getter_doc_override(self): + newgettersub = PropertySubNewGetter() + self.assertEqual(newgettersub.spam, 5) + self.assertEqual(newgettersub.__class__.spam.__doc__, "new docstring") + newgetter = PropertyNewGetter() + self.assertEqual(newgetter.spam, 8) + self.assertEqual(newgetter.__class__.spam.__doc__, "new docstring") # def test_property___isabstractmethod__descriptor(self): # for val in (True, False, [], [1], '', '1'): @@ -145,24 +145,24 @@ def test_property_decorator_subclass(self): # foo = property(foo) # C.foo.__isabstractmethod__ - # def test_property_builtin_doc_writable(self): - # p = property(doc='basic') - # self.assertEqual(p.__doc__, 'basic') - # p.__doc__= 'extended' - # self.assertEqual(p.__doc__, 'extended') - - # def test_property_decorator_doc_writable(self): - # class PropertyWritableDoc(object): - # - # @property - # def spam(self): - # """Eggs""" - # return "eggs" - # - # sub = PropertyWritableDoc() - # self.assertEqual(sub.__class__.spam.__doc__, 'Eggs') - # sub.__class__.spam.__doc__ = 'Spam' - # self.assertEqual(sub.__class__.spam.__doc__, 'Spam') + def test_property_builtin_doc_writable(self): + p = property(doc='basic') + self.assertEqual(p.__doc__, 'basic') + p.__doc__= 'extended' + self.assertEqual(p.__doc__, 'extended') + + def test_property_decorator_doc_writable(self): + class PropertyWritableDoc(object): + + @property + def spam(self): + """Eggs""" + return "eggs" + + sub = PropertyWritableDoc() + self.assertEqual(sub.__class__.spam.__doc__, 'Eggs') + sub.__class__.spam.__doc__ = 'Spam' + self.assertEqual(sub.__class__.spam.__doc__, 'Spam') # Issue 5890: subclasses of property do not preserve method __doc__ strings class PropertySub(property): @@ -191,68 +191,68 @@ def spam(self, value): class PropertySubclassTests(unittest.TestCase): - # def test_slots_docstring_copy_exception(self): - # try: - # class Foo(object): - # @PropertySubSlots - # def spam(self): - # """Trying to copy this docstring will raise an exception""" - # return 1 - # except AttributeError: - # pass - # else: - # raise Exception("AttributeError not raised") - # - # def test_docstring_copy(self): - # class Foo(object): - # @PropertySub - # def spam(self): - # """spam wrapped in property subclass""" - # return 1 - # self.assertEqual( - # Foo.spam.__doc__, - # "spam wrapped in property subclass") + def test_slots_docstring_copy_exception(self): + try: + class Foo(object): + @PropertySubSlots + def spam(self): + """Trying to copy this docstring will raise an exception""" + return 1 + except AttributeError: + pass + else: + raise Exception("AttributeError not raised") + + def test_docstring_copy(self): + class Foo(object): + @PropertySub + def spam(self): + """spam wrapped in property subclass""" + return 1 + self.assertEqual( + Foo.spam.__doc__, + "spam wrapped in property subclass") def test_property_setter_copies_getter_docstring(self): foo = Foo() self.assertEqual(foo.spam, 1) foo.spam = 2 self.assertEqual(foo.spam, 2) - # self.assertEqual( - # Foo.spam.__doc__, - # "spam wrapped in property subclass") + self.assertEqual( + Foo.spam.__doc__, + "spam wrapped in property subclass") foosub = FooSub() self.assertEqual(foosub.spam, 1) foosub.spam = 7 self.assertEqual(foosub.spam, 'eggs') - # self.assertEqual( - # FooSub.spam.__doc__, - # "spam wrapped in property subclass") - - # def test_property_new_getter_new_docstring(self): - # - # class Foo(object): - # @PropertySub - # def spam(self): - # """a docstring""" - # return 1 - # @spam.getter - # def spam(self): - # """a new docstring""" - # return 2 - # self.assertEqual(Foo.spam.__doc__, "a new docstring") - # class FooBase(object): - # @PropertySub - # def spam(self): - # """a docstring""" - # return 1 - # class Foo2(FooBase): - # @FooBase.spam.getter - # def spam(self): - # """a new docstring""" - # return 2 - # self.assertEqual(Foo.spam.__doc__, "a new docstring") + self.assertEqual( + FooSub.spam.__doc__, + "spam wrapped in property subclass") + + def test_property_new_getter_new_docstring(self): + + class Foo(object): + @PropertySub + def spam(self): + """a docstring""" + return 1 + @spam.getter + def spam(self): + """a new docstring""" + return 2 + self.assertEqual(Foo.spam.__doc__, "a new docstring") + # class FooBase(object): + # @PropertySub + # def spam(self): + # """a docstring""" + # return 1 + # class Foo2(FooBase): + # @FooBase.spam.getter + # def spam(self): + # """a new docstring""" + # return 2 + # self.assertEqual(Foo.spam.__doc__, "a new docstring") class TestDecorators(unittest.TestCase): From 5c3c9e1275e6b25a32c7bdf5b0909d13ca42d2bd Mon Sep 17 00:00:00 2001 From: stu Date: Tue, 10 Oct 2023 12:34:40 +0800 Subject: [PATCH 131/137] dict: fix setattr should not call mp$ass_subscript on the `__dict__` --- src/dict.js | 35 +++++++++++++++++++++------------- src/generic.js | 19 ++++++++---------- test/unit3/test_skulpt_bugs.py | 25 ++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 24 deletions(-) diff --git a/src/dict.js b/src/dict.js index 0a088a949c..b38914c5f8 100644 --- a/src/dict.js +++ b/src/dict.js @@ -25,7 +25,7 @@ Sk.builtin.dict = Sk.abstr.buildNativeClass("dict", { this.entries = Object.create(null); this.buckets = {}; for (let i = 0; i < L.length; i += 2) { - this.set$item(L[i], L[i + 1]); + this.dict$setItem(L[i], L[i + 1]); } this.in$repr = false; this.$version = 0; // change version number anytime the keys change @@ -116,12 +116,12 @@ Sk.builtin.dict = Sk.abstr.buildNativeClass("dict", { }, mp$ass_subscript(key, value) { if (value === undefined) { - const item = this.pop$item(key); - if (item === undefined) { - throw new Sk.builtin.KeyError(key); + const err = this.dict$delItem(key); + if (err) { + throw err; } } else { - this.set$item(key, value); + this.dict$setItem(key, value); } }, }, @@ -312,7 +312,8 @@ Sk.builtin.dict = Sk.abstr.buildNativeClass("dict", { $items() { return Object.values(this.entries); }, - set$item, + dict$setItem, + dict$delItem, get$bucket_item, pop$bucket_item, set$bucket_item, @@ -487,18 +488,18 @@ function mp$lookup(key) { * almost similar code, this may be changed in future * * Note we don't use mp$ass_subscript since that slot might be overridden by a subclass - * Instead we use this.set$item which is the dict implementation of mp$ass_subscript + * Instead we use this.dict$setItem which is the dict implementation of mp$ass_subscript * @private */ function dict$merge(b) { // we don't use mp$ass_subscript incase a subclass overrides __setitem__ we just ignore that like Cpython does - // so use this.set$item instead which can't be overridden by a subclass + // so use this.dict$setItem instead which can't be overridden by a subclass if (b.tp$iter === Sk.builtin.dict.prototype.tp$iter) { // fast way used const keys = b.tp$iter(); for (let key = keys.tp$iternext(); key !== undefined; key = keys.tp$iternext()) { const v = b.mp$subscript(key); - this.set$item(key, v); + this.dict$setItem(key, v); } } else { // generic slower way for a subclass that has overriden the tp$iter method @@ -511,7 +512,7 @@ function dict$merge(b) { return Sk.misceval.chain(Sk.misceval.callsimOrSuspendArray(keyfunc), (keys) => Sk.misceval.iterFor(Sk.abstr.iter(keys), (key) => Sk.misceval.chain(Sk.abstr.objectGetItem(b, key, true), (v) => { - this.set$item(key, v); + this.dict$setItem(key, v); }) ) ); @@ -551,7 +552,7 @@ function update$common(args, kwargs, func_name) { return Sk.misceval.chain(ret, () => { if (kwargs) { for (let i = 0; i < kwargs.length; i += 2) { - this.set$item(new Sk.builtin.str(kwargs[i]), kwargs[i + 1]); + this.dict$setItem(new Sk.builtin.str(kwargs[i]), kwargs[i + 1]); } } return; @@ -580,7 +581,7 @@ function dict$merge_seq(arg) { if (seq.length !== 2) { throw new Sk.builtin.ValueError("dictionary update sequence element #" + idx + " has length " + seq.length + "; 2 is required"); } - this.set$item(seq[0], seq[1]); + this.dict$setItem(seq[0], seq[1]); idx++; }); }; @@ -596,7 +597,7 @@ function dict$merge_seq(arg) { * @private * */ -function set$item(key, value) { +function dict$setItem(key, value) { const hash = getHash(key); let item; if (typeof hash === "string") { @@ -621,6 +622,14 @@ function set$item(key, value) { } }; +/** means we don't need to wrap in a try/except for genericSetAttr */ +function dict$delItem(key) { + const item = this.pop$item(key); + if (item === undefined) { + return new Sk.builtin.KeyError(key); + } +} + /** * @function * diff --git a/src/generic.js b/src/generic.js index b010b719e7..dacb598291 100644 --- a/src/generic.js +++ b/src/generic.js @@ -83,21 +83,18 @@ Sk.generic.setAttr = function __setattr__(pyName, value, canSuspend) { } const dict = this.$d; - if (dict !== undefined) { - if (dict.mp$ass_subscript) { + if (dict !== undefined && dict !== null) { + if (dict.dict$setItem) { if (value !== undefined) { - return dict.mp$ass_subscript(pyName, value); + return dict.dict$setItem(pyName, value); } else { - try { - return dict.mp$ass_subscript(pyName); - } catch (e) { - if (e instanceof Sk.builtin.KeyError) { - throw new Sk.builtin.AttributeError("'" + Sk.abstr.typeName(this) + "' object has no attribute '" + pyName.$jsstr() + "'"); - } - throw e; + const err = dict.dict$delItem(pyName); + if (err) { + throw new Sk.builtin.AttributeError("'" + Sk.abstr.typeName(this) + "' object has no attribute '" + pyName.$jsstr() + "'"); } + return; } - } else if (typeof dict === "object") { + } else if (typeof dict === "object" && !dict.sk$object) { const jsMangled = pyName.$mangled; if (value !== undefined) { dict[jsMangled] = value; diff --git a/test/unit3/test_skulpt_bugs.py b/test/unit3/test_skulpt_bugs.py index ddac42494c..72884408aa 100644 --- a/test/unit3/test_skulpt_bugs.py +++ b/test/unit3/test_skulpt_bugs.py @@ -50,5 +50,30 @@ def test_bug_1428(self): annotations = An.__annotations__ self.assertEqual(annotations, {"name": str, "_An__foo": int}) + +class TestDict(unittest.TestCase): + def test_override_dunder_dict(self): + class D(dict): + called = False + def __getitem__(self, key): + self.called = True + return dict.__getitem__(self, key) + + def __setitem__(self, key, value): + self.called = True + return dict.__setitem__(self, key, value) + + class A: + def __init__(self): + self.__dict__ = D() + + a = A() + + self.assertFalse(a.__dict__.called) + a.foo = "bar" + self.assertFalse(a.__dict__.called) + self.assertEqual(a.foo, "bar") + self.assertFalse(a.__dict__.called) + if __name__ == "__main__": unittest.main() From 2c7130e7fad245d6eedb7f43f3a5ef59bfb9f447 Mon Sep 17 00:00:00 2001 From: stu Date: Wed, 11 Oct 2023 22:43:08 +0800 Subject: [PATCH 132/137] Fix unreserve words to __set_name__ --- src/compile.js | 3 ++- src/type.js | 5 ++-- test/unit/test_reserved_words.py | 40 ++++++++++++++++++++++++-------- 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/src/compile.js b/src/compile.js index 451593513f..9af6e77804 100644 --- a/src/compile.js +++ b/src/compile.js @@ -136,8 +136,9 @@ function fixReserved(name) { return name + "_$rw$"; } +const reservedSuffix = /_\$rw\$$/; function unfixReserved(name) { - return name.replace(/_\$rw\$$/, ""); + return name.replace(reservedSuffix, ""); } function mangleName (priv, ident) { diff --git a/src/type.js b/src/type.js index 74ef6bbb69..05c647de6d 100644 --- a/src/type.js +++ b/src/type.js @@ -828,11 +828,12 @@ function set_names(type) { Object.keys(proto).forEach((key) => { const set_func = Sk.abstr.lookupSpecial(proto[key], Sk.builtin.str.$setname); if (set_func !== undefined) { + const name = Sk.unfixReserved(key); try { - Sk.misceval.callsimArray(set_func, [type, new Sk.builtin.str(key)]); + Sk.misceval.callsimArray(set_func, [type, new Sk.builtin.str(name)]); } catch (e) { const runtime_err = new Sk.builtin.RuntimeError( - "Error calling __set_name__ on '" + Sk.abstr.typeName(proto[key]) + "' instance '" + key + "' in '" + type.prototype.tp$name + "'" + "Error calling __set_name__ on '" + Sk.abstr.typeName(proto[key]) + "' instance '" + name + "' in '" + type.prototype.tp$name + "'" ); runtime_err.$cause = e; throw runtime_err; diff --git a/test/unit/test_reserved_words.py b/test/unit/test_reserved_words.py index 5edaa7c9af..599c58c308 100644 --- a/test/unit/test_reserved_words.py +++ b/test/unit/test_reserved_words.py @@ -1,19 +1,27 @@ import unittest -class TestFixture(): + +class TestFixture: def default(self): return True def delete(self): return True +class Prop: + def __set_name__(self, owner, name): + self._name = name + class A: pass + + a = A() -a.name = 'foo' -a.length = 'bar' +a.name = "foo" +a.length = "bar" default = "foo" + class Test_ReservedWords(unittest.TestCase): def test_getattr(self): f = TestFixture() @@ -27,8 +35,8 @@ def test_getattr(self): self.assertNotIn("$", repr(func_delete)) def test_getattr_with_name(self): - self.assertEqual(getattr(a, 'name'), 'foo') - self.assertEqual(getattr(a, 'length'), 'bar') + self.assertEqual(getattr(a, "name"), "foo") + self.assertEqual(getattr(a, "length"), "bar") def test_setattr(self): f = TestFixture() @@ -48,26 +56,38 @@ def test_error_message(self): try: f.name except AttributeError as e: - self.assertTrue('_$rn$' not in str(e)) + self.assertTrue("_$rn$" not in str(e)) def test_dir(self): - self.assertTrue('name' in dir(a)) - self.assertTrue('length' in dir(a)) + self.assertTrue("name" in dir(a)) + self.assertTrue("length" in dir(a)) def test_arguments_prototype(self): with self.assertRaises(AttributeError): A.prototype with self.assertRaises(AttributeError): - A.arguments + A.arguments def test_bug_949(self): def wrapper(name): def inner(): return name + return inner + self.assertEqual(wrapper("foo")(), "foo") + def test_global(self): self.assertTrue("default" in globals()) -if __name__ == '__main__': + def test_setname(self): + + + class Foo: + length = Prop() + + self.assertEqual(Foo.length._name, "length") + + +if __name__ == "__main__": unittest.main() From 35e4a225b4fbef895b7c3baf1074fc05cd74742d Mon Sep 17 00:00:00 2001 From: stu Date: Thu, 12 Oct 2023 15:28:15 +0800 Subject: [PATCH 133/137] ffi: wrap symbols --- src/ffi.js | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/ffi.js b/src/ffi.js index 306f9a1426..1563f1d985 100644 --- a/src/ffi.js +++ b/src/ffi.js @@ -57,6 +57,8 @@ function toPy(obj, hooks) { if (type === "string") { return new Sk.builtin.str(obj); + } else if (type === "symbol") { + return new WrappedSymbol(obj); } else if (type === "number") { return numberToPy(obj); } else if (type === "boolean") { @@ -156,6 +158,8 @@ function toJs(obj, hooks) { if (type === "string") { return hooks.stringHook ? hooks.stringHook(val) : val; + } else if (type === "symbol") { + return val; } else if (type === "boolean") { return val; } else if (type === "number") { @@ -739,3 +743,23 @@ function checkBodyIsMaybeConstructor(obj) { return !noNewNeeded.has(obj); } } + + +const WrappedSymbol = Sk.abstr.buildNativeClass("ProxySymbol", { + constructor: function WrappedSymbol(symbol) { + this.v = symbol; + }, + slots: { + $r() { + return new Sk.builtin.str(this.toString()); + } + }, + proto: { + toString() { + return this.v.toString(); + }, + valueOf() { + return this.v; + } + } +}); From e3e46ebeb4e7eb716cb5f7f21c36cec9ce9716a9 Mon Sep 17 00:00:00 2001 From: stu Date: Sun, 15 Oct 2023 14:11:47 +0800 Subject: [PATCH 134/137] ffi: preserve arrays on javascript objects --- src/ffi.js | 105 ++++++++++++++++++++++++++---- test/unit3/test_skulpt_interop.py | 44 +++++++++++++ 2 files changed, 135 insertions(+), 14 deletions(-) create mode 100644 test/unit3/test_skulpt_interop.py diff --git a/src/ffi.js b/src/ffi.js index 1563f1d985..456bbf1ea9 100644 --- a/src/ffi.js +++ b/src/ffi.js @@ -71,7 +71,7 @@ function toPy(obj, hooks) { // might be type === "bigint" if bigint native or an array like object for older browsers return new Sk.builtin.int_(JSBI.numberIfSafe(obj)); } else if (Array.isArray(obj)) { - return new Sk.builtin.list(obj.map((x) => toPy(x, hooks))); + return hooks.arrayHook ? hooks.arrayHook(obj) : new Sk.builtin.list(obj.map((x) => toPy(x, hooks))); } else if (type === "object") { const constructor = obj.constructor; // it's possible that a library deleted the constructor if (constructor === Object && Object.getPrototypeOf(obj) === OBJECT_PROTO || constructor === undefined /* Object.create(null) */) { @@ -323,17 +323,29 @@ function proxy(obj, flags) { flags.name = cached.$name; } } - const ret = new JsProxy(obj, flags); - _proxied.set(obj, ret); - return ret; + let rv; + if (Array.isArray(obj)) { + rv = new JsProxyList(obj); + } else { + rv = new JsProxy(obj, flags); + } + _proxied.set(obj, rv); + return rv; } -const pyHooks = { dictHook: (obj) => proxy(obj), unhandledHook: (obj) => String(obj) }; +const dictHook = (obj) => proxy(obj); +const unhandledHook = (obj) => String(obj); +const arrayHook = (obj) => proxy(obj); + +const pyHooks = { arrayHook, dictHook, unhandledHook }; + + // unhandled is likely only Symbols and get a string rather than undefined const boundHook = (bound, name) => ({ - dictHook: (obj) => proxy(obj), + dictHook, funcHook: (obj) => proxy(obj, { bound, name }), - unhandledHook: (obj) => String(obj), + unhandledHook, + arrayHook, }); const jsHooks = { unhandledHook: (obj) => { @@ -374,6 +386,15 @@ const jsHooks = { // we customize the dictHook and the funcHook here - we want to keep object literals as proxied objects when remapping to Py // and we want funcs to be proxied +function setJsProxyAttr(pyName, pyValue) { + const jsName = pyName.toString(); + if (pyValue === undefined) { + delete this.js$wrapped[jsName]; + } else { + this.js$wrapped[jsName] = toJs(pyValue, jsHooks); + } +} + const JsProxy = Sk.abstr.buildNativeClass("Proxy", { constructor: function JsProxy(obj, flags) { if (obj === undefined) { @@ -412,13 +433,8 @@ const JsProxy = Sk.abstr.buildNativeClass("Proxy", { tp$getattr(pyName) { return this.$lookup(pyName) || Sk.generic.getAttr.call(this, pyName); }, - tp$setattr(pyName, value) { - const jsName = pyName.toString(); - if (value === undefined) { - delete this.js$wrapped[jsName]; - } else { - this.js$wrapped[jsName] = toJs(value, jsHooks); - } + tp$setattr(pyName, pyValue) { + return setJsProxyAttr.call(this, pyName, pyValue); }, $r() { if (this.is$callable) { @@ -721,6 +737,67 @@ const JsProxy = Sk.abstr.buildNativeClass("Proxy", { }, }); +const ArrayFunction = { + apply(target, thisArg, argumentsList) { + const jsArgs = toJsArray(argumentsList, jsHooks); + return target.apply(thisArg, jsArgs); + } +}; + +const arrayMethods = {}; +const ArrayProto = Array.prototype; + +const arrayHandler = { + get(target, attr) { + const rv = target[attr]; + if (attr in ArrayProto) { + // internal calls like this.v.pop(); this.v.push(x); + if (typeof rv === "function") { + return (arrayMethods[attr] || (arrayMethods[attr] = new Proxy(rv, ArrayFunction))); + } + // this.v.length; + return rv; + } + if (rv === undefined && !(attr in target)) { + return rv; + } + console.log(attr, target); + debugger; + // attributes on the list instance; + return toPy(rv, pyHooks); + }, + set(target, attr, value) { + // for direct access of the array via this.v[x] = y; + target[attr] = toJs(value, jsHooks); + return true; + } +}; + +const JsProxyList = Sk.abstr.buildNativeClass("ProxyList", { + base: Sk.builtin.list, + constructor: function (L) { + Sk.builtin.list.call(this, L); + this.js$wrapped = this.v; + this.v = new Proxy(this.v, arrayHandler); + }, + slots: { + tp$getattr(pyName) { + // python methods win + return Sk.generic.getAttr.call(this, pyName) || this.$lookup(pyName); + }, + tp$setattr(pyName, pyValue) { + return setJsProxyAttr.call(this, pyName, pyValue); + }, + $r() { + return new Sk.builtin.str("proxylist(" + Sk.builtin.list.prototype.$r.call(this) + ")"); + } + }, + proto: { + $lookup: JsProxy.prototype.$lookup, + } +}); + + const is_constructor = /^class|^function[a-zA-Z\d\(\)\{\s]+\[native code\]\s+\}$/; const getFunctionBody = FUNC_PROTO.toString; diff --git a/test/unit3/test_skulpt_interop.py b/test/unit3/test_skulpt_interop.py new file mode 100644 index 0000000000..51bba32653 --- /dev/null +++ b/test/unit3/test_skulpt_interop.py @@ -0,0 +1,44 @@ +"""Unit tests for zero-argument super() & related machinery.""" + +import unittest + + +window = jseval("self") + +class TestProxyArray(unittest.TestCase): + def test_basic(self): + x = [1, 2, 3] + window.x = x + self.assertIsNot(x, window.x) + self.assertEqual(x, window.x) + + x = window.x + self.assertIs(x, window.x) + + self.assertIsInstance(x, list) + x.append(4) + x.push(5) + self.assertEqual(x, [1, 2, 3, 4, 5]) + self.assertEqual(len(x), 5) + + c = x.copy() + self.assertIs(type(c), list) + self.assertEqual(c, x) + + x.foo = "bar" + self.assertEqual(x.foo, "bar") + + r = x.splice(3, 2) + self.assertEqual(r, [4, 5]) + + r = x.map(lambda v, *args: v + 1) + self.assertEqual(r, [v + 1 for v in x]) + + i = x.pop() + self.assertEqual(i, 3) + i = x.pop(0) # use python method over js method here + self.assertEqual(i, 1) + + +if __name__ == "__main__": + unittest.main() From d71b61de7f6531c9bd23900f98a9047b133237be Mon Sep 17 00:00:00 2001 From: stu Date: Sun, 15 Oct 2023 14:54:01 +0800 Subject: [PATCH 135/137] use Sk.global instead --- test/unit3/test_skulpt_interop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit3/test_skulpt_interop.py b/test/unit3/test_skulpt_interop.py index 51bba32653..b91e2c077e 100644 --- a/test/unit3/test_skulpt_interop.py +++ b/test/unit3/test_skulpt_interop.py @@ -3,7 +3,7 @@ import unittest -window = jseval("self") +window = jseval("Sk.global") class TestProxyArray(unittest.TestCase): def test_basic(self): From 3afaf50545dd2243041da82f80d2e7202cd4847a Mon Sep 17 00:00:00 2001 From: stu Date: Sun, 15 Oct 2023 15:05:05 +0800 Subject: [PATCH 136/137] Remove debugging --- src/ffi.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/ffi.js b/src/ffi.js index 456bbf1ea9..6dc81396b8 100644 --- a/src/ffi.js +++ b/src/ffi.js @@ -761,8 +761,6 @@ const arrayHandler = { if (rv === undefined && !(attr in target)) { return rv; } - console.log(attr, target); - debugger; // attributes on the list instance; return toPy(rv, pyHooks); }, From c0ee3a8f273a892ee10246ecbccdc105d34ea38d Mon Sep 17 00:00:00 2001 From: stu Date: Mon, 16 Oct 2023 12:04:24 +0800 Subject: [PATCH 137/137] Minor tweak to array check --- src/ffi.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ffi.js b/src/ffi.js index 6dc81396b8..f928fabd72 100644 --- a/src/ffi.js +++ b/src/ffi.js @@ -303,7 +303,7 @@ function toPyDict(obj, hooks) { // cache the proxied objects in a weakmap const _proxied = new WeakMap(); -// use proxy if you want to proxy an arbirtrary js object +// use proxy if you want to proxy an arbitrary js object // the only flags currently used is {bound: some_js_object} function proxy(obj, flags) { if (obj === null || obj === undefined) { @@ -324,7 +324,7 @@ function proxy(obj, flags) { } } let rv; - if (Array.isArray(obj)) { + if (type !== "function" && Array.isArray(obj)) { rv = new JsProxyList(obj); } else { rv = new JsProxy(obj, flags);