ModuleNotFoundError in doctesting src/sage/misc/sageinspect.py

[strogdon]

The two main failures are:

File "src/sage/misc/sageinspect.py", line 2102, in sage.misc.sageinspect._sage_getsourcelines_name_with_dot
Failed example:
    cython('''
    class A:
        def __init__(self):
            "some init doc"
            pass
    class B:
        "some class doc"
        class A(A):
            pass
    ''')
Exception raised:
    Traceback (most recent call last):
      File "/local/sage-git/sage/local/lib/python3.9/site-packages/sage/doctest/forker.py", line 714, in _run
        self.compile_and_execute(example, compiler, test.globs)
      File "/local/sage-git/sage/local/lib/python3.9/site-packages/sage/doctest/forker.py", line 1133, in compile_and_execute
        exec(compiled, globs)
      File "<doctest sage.misc.sageinspect._sage_getsourcelines_name_with_dot[3]>", line 1, in <module>
        cython('''
      File "sage/misc/lazy_import.pyx", line 360, in sage.misc.lazy_import.LazyImport.__call__ (build/cythonized/sage/misc/lazy_import.c:4032)
        return self.get_object()(*args, **kwds)
      File "/local/sage-git/sage/local/lib/python3.9/site-packages/sage/misc/cython.py", line 659, in cython_compile
        return cython_import_all(tmpfile, get_globals(), **kwds)
      File "/local/sage-git/sage/local/lib/python3.9/site-packages/sage/misc/cython.py", line 549, in cython_import_all
        m = cython_import(filename, **kwds)
      File "/local/sage-git/sage/local/lib/python3.9/site-packages/sage/misc/cython.py", line 529, in cython_import
        return builtins.__import__(name)
    ModuleNotFoundError: No module named '_home_steven__sage_temp_hp_probook_2359_tmp_isl9jpcz_pyx_0'

and

File "src/sage/misc/sageinspect.py", line 2251, in sage.misc.sageinspect.sage_getsourcelines
Failed example:
    cython('''cpdef test_funct(x,y): return''')
Exception raised:
    Traceback (most recent call last):
      File "/local/sage-git/sage/local/lib/python3.9/site-packages/sage/doctest/forker.py", line 714, in _run
        self.compile_and_execute(example, compiler, test.globs)
      File "/local/sage-git/sage/local/lib/python3.9/site-packages/sage/doctest/forker.py", line 1133, in compile_and_execute
        exec(compiled, globs)
      File "<doctest sage.misc.sageinspect.sage_getsourcelines[6]>", line 1, in <module>
        cython('''cpdef test_funct(x,y): return''')
      File "sage/misc/lazy_import.pyx", line 360, in sage.misc.lazy_import.LazyImport.__call__ (build/cythonized/sage/misc/lazy_import.c:4032)
        return self.get_object()(*args, **kwds)
      File "/local/sage-git/sage/local/lib/python3.9/site-packages/sage/misc/cython.py", line 659, in cython_compile
        return cython_import_all(tmpfile, get_globals(), **kwds)
      File "/local/sage-git/sage/local/lib/python3.9/site-packages/sage/misc/cython.py", line 549, in cython_import_all
        m = cython_import(filename, **kwds)
      File "/local/sage-git/sage/local/lib/python3.9/site-packages/sage/misc/cython.py", line 529, in cython_import
        return builtins.__import__(name)
    ModuleNotFoundError: No module named '_home_steven__sage_temp_hp_probook_2359_tmp__qk8dlm5_pyx_0'

Hardware:

Linux hp-probook 5.4.97-gentoo-x86_64 #16 SMP Mon Mar 15 21:41:09 MDT 2021 x86_64 AMD Ryzen 7 4700U with Radeon Graphics AuthenticAMD GNU/Linux

04:00.0 Non-Volatile memory controller: Intel Corporation SSD 660P Series (rev 03) (prog-if 02 [NVM Express])
        Subsystem: Intel Corporation SSD 660P Series

[strogdon]

A simple example that demonstrates the failure is:

sage: cython(''' 
....: ''')                                                                                                                                                                                                                                   
---------------------------------------------------------------------------
ModuleNotFoundError                       Traceback (most recent call last)
<ipython-input-1-e3fd0203ef75> in <module>
----> 1 cython('''
      2 ''')

/local/sage-git/sage/local/lib/python3.9/site-packages/sage/misc/lazy_import.pyx in sage.misc.lazy_import.LazyImport.__call__ (build/cythonized/sage/misc/lazy_import.c:4032)()
    358             True
    359         """
--> 360         return self.get_object()(*args, **kwds)
    361 
    362     def __repr__(self):

/local/sage-git/sage/local/lib/python3.9/site-packages/sage/misc/cython.py in cython_compile(code, **kwds)
    657     with open(tmpfile, 'w') as f:
    658         f.write(code)
--> 659     return cython_import_all(tmpfile, get_globals(), **kwds)
    660 
    661 

/local/sage-git/sage/local/lib/python3.9/site-packages/sage/misc/cython.py in cython_import_all(filename, globals, **kwds)
    547       code
    548     """
--> 549     m = cython_import(filename, **kwds)
    550     for k, x in m.__dict__.items():
    551         if k[0] != '_':

/local/sage-git/sage/local/lib/python3.9/site-packages/sage/misc/cython.py in cython_import(filename, **kwds)
    527     try:
    528         sys.path.append(build_dir)
--> 529         return builtins.__import__(name)
    530     finally:
    531         sys.path = oldpath

ModuleNotFoundError: No module named '_home_steven__sage_temp_hp_probook_3426_tmp_p61fqom7_pyx_0'

where the module is present

$ tree -a ~/.sage/temp
/home/steven/.sage/temp
└── hp-probook
    ├── 3426
    │   ├── ecl
    │   ├── spyx
    │   │   └── _home_steven__sage_temp_hp_probook_3426_tmp_p61fqom7_pyx
    │   │       ├── build
    │   │       │   └── temp.linux-x86_64-3.9
    │   │       │       └── home
    │   │       │           └── steven
    │   │       │               └── .sage
    │   │       │                   └── temp
    │   │       │                       └── hp-probook
    │   │       │                           └── 3426
    │   │       │                               └── spyx
    │   │       │                                   └── _home_steven__sage_temp_hp_probook_3426_tmp_p61fqom7_pyx
    │   │       │                                       └── _home_steven__sage_temp_hp_probook_3426_tmp_p61fqom7_pyx_0.o
    │   │       ├── _home_steven__sage_temp_hp_probook_3426_tmp_p61fqom7_pyx_0.c
    │   │       ├── _home_steven__sage_temp_hp_probook_3426_tmp_p61fqom7_pyx_0.cpython-39-x86_64-linux-gnu.so
    │   │       ├── _home_steven__sage_temp_hp_probook_3426_tmp_p61fqom7_pyx_0.err
    │   │       ├── _home_steven__sage_temp_hp_probook_3426_tmp_p61fqom7_pyx_0.html
    │   │       ├── _home_steven__sage_temp_hp_probook_3426_tmp_p61fqom7_pyx_0.lis
    │   │       └── _home_steven__sage_temp_hp_probook_3426_tmp_p61fqom7_pyx_0.pyx
    │   └── tmp_p61fqom7.pyx
    ├── cleaner.log
    └── cleaner.pid

Occasionally, the above will not fail.

[strogdon]

Usually the following will not fail

sage: cython(''' 
....: ''', create_local_so_file=True)
sage:

but even this will occasionally fail.

[mkoeppe]

see also old ticket #28928

[strogdon]

This hack seems to resolve things here

src/sage/misc/cython.py

@@ -523 +523 @@
     """
     name, build_dir = cython(filename, **kwds)
 
+    build_dir = build_dir + '/'
+
     oldpath = sys.path
     try:
         sys.path.append(build_dir)

$ ./sage -t src/sage/misc/sageinspect.py 
Running doctests with ID 2021-04-23-20-30-52-2178c446.
Git branch: develop
Using --optional=build,dochtml,gentoo,pip,sage,sage_spkg
Doctesting 1 file.
sage -t --warn-long 95.9 --random-seed=0 src/sage/misc/sageinspect.py
    [340 tests, 106.40 s]
----------------------------------------------------------------------
All tests passed!
----------------------------------------------------------------------
Total time for all tests: 106.7 seconds
    cpu time: 103.9 seconds
    cumulative wall time: 106.4 seconds

And the ​above simple example:

sage: import sys                                                                                         
sage: cython(''' 
....: ''')                                                                                               
sage: sys.path                                                                                           
['/local/sage-git/sage/src/bin',
 '/usr/lib/python39.zip',
 '/usr/lib/python3.9',
 '/usr/lib/python3.9/lib-dynload',
 '',
 '/local/sage-git/sage/local/lib/python3.9/site-packages',
 '/local/sage-git/sage/local/lib/python3.9/site-packages/IPython/extensions',
 '/home/steven/.sage/ipython-5.0.0',
 '/home/steven/.sage/temp/hp-probook/10953/spyx/_home_steven__sage_temp_hp_probook_10953_tmp_terjz7ms_pyx/']
sage:

So why is the / needed?

[gh-sheerluck]

"three dots magic" is too slow, it costs 130 seconds:

src/sage/misc/sageinspect.py

@@ -2276 +2276 @@
         ([...'class MPolynomialIdeal( MPolynomialIdeal_singular_repr, \\\n',
         ...)
         sage: x = var('x')
-        sage: sage_getsourcelines(x)
-        (['cdef class Expression(CommutativeRingElement):\n',
-          '    cpdef object pyobject(self):\n',
-        ...)
-        sage: sage_getsourcelines(x)[0][-1]    # last line
+        sage: lines, lineno = sage_getsourcelines(x); lines[0:2]
+        ['cdef class Expression(CommutativeRingElement):\n',
+         '    cpdef object pyobject(self):\n']
+        sage: lines[-1]    # last line
         '        return S\n'

before:

sage -t --long --random-seed=0 src/sage/misc/sageinspect.py
    [340 tests, 158.98 s]

after:

sage -t --long --random-seed=0 src/sage/misc/sageinspect.py
    [340 tests, 34.82 s]

[strogdon]

Same basic result here on hardware where there is no ModuleNotFoundError:

before:

sage -t --long --warn-long 182.5 --random-seed=0 src/sage/misc/sageinspect.py
    [340 tests, 156.74 s]

after:

sage -t --long --warn-long 183.3 --random-seed=0 src/sage/misc/sageinspect.py
    [340 tests, 25.88 s]

However, this does not resolve the ModuleNotFoundError on the hardware listed in the description.

[strogdon]

Invalidating internal caches seems to resolve things here on the machine in the description (see ​https://docs.python.org/3/library/importlib.html#importlib.invalidate_caches

src/sage/misc/cython.py

@@ -526 +526 @@
     oldpath = sys.path
     try:
         sys.path.append(build_dir)
+        import importlib
+        importlib.invalidate_caches()
         return builtins.__import__(name)
     finally:
         sys.path = oldpath

I'm not sure why it's needed nor if this is a proper fix, but sageinspect.py does not fail with it

$ ./sage -t src/sage/misc/sageinspect.py 
Running doctests with ID 2021-06-17-15-51-51-0193a7dc.
Git branch: trac_31626
Using --optional=build,dochtml,gentoo,pip,sage,sage_spkg
Doctesting 1 file.
sage -t --warn-long 95.3 --random-seed=0 src/sage/misc/sageinspect.py
    [340 tests, 152.35 s]
----------------------------------------------------------------------
All tests passed!
----------------------------------------------------------------------
Total time for all tests: 152.6 seconds
    cpu time: 149.9 seconds
    cumulative wall time: 152.4 seconds
Pytest is not installed, skip checking tests that rely on it.

[fbissey]

I actually like it. From ​https://docs.python.org/3/library/importlib.html :

"This function should be called if any modules are created/installed while your program is running to guarantee all finders will notice the new module’s existence."

So, it is actually highly appropriate because that's exactly what the code is doing. Create a temporary module and trying to load it.

[strogdon]

Pushed the recent suggestion. Let's see what the Bots say - if there are any unintended consequences. This may be difficult to review since the issue seems rare.

---

New commits:

​926359e

Fix for when cython created temporary modules throw a

[strogdon]

Not every meta path finder

sage: sys.meta_path                                                                                      
[<class '_frozen_importlib.BuiltinImporter'>,
 <class '_frozen_importlib.FrozenImporter'>,
 <class '_frozen_importlib_external.PathFinder'>,
 <six._SixMetaPathImporter object at 0x7fd4688217c0>]

has an invalidate_caches attribute, that allows applying the invalidate_caches() method. The offending item here is PathFinder.

sage: import _frozen_importlib_external                                                                  
sage: _frozen_importlib_external.PathFinder.__dict__                                                     
mappingproxy({'__module__': '_frozen_importlib_external',
              '__doc__': 'Meta path finder for sys.path and package __path__ attributes.',
              'invalidate_caches': <classmethod object at 0x7fd468c713a0>,
              '_path_hooks': <classmethod object at 0x7fd468c713d0>,
              '_path_importer_cache': <classmethod object at 0x7fd468c71400>,
              '_legacy_get_spec': <classmethod object at 0x7fd468c71430>,
              '_get_spec': <classmethod object at 0x7fd468c71460>,
              'find_spec': <classmethod object at 0x7fd468c71490>,
              'find_module': <classmethod object at 0x7fd468c714c0>,
              'find_distributions': <classmethod object at 0x7fd468c714f0>,
              '__dict__': <attribute '__dict__' of 'PathFinder' objects>,
              '__weakref__': <attribute '__weakref__' of 'PathFinder' objects>})

The following achieves the same as this branch

src/sage/misc/cython.py

@@ -526 +526 @@
     oldpath = sys.path
     try:
         sys.path.append(build_dir)
+        import _frozen_importlib_external
+        _frozen_importlib_external.PathFinder.invalidate_caches()
         return builtins.__import__(name)
     finally:
         sys.path = oldpath

Apparently, the ModuleNotFoundError is the result of a race condition that can exit on certain archs when a module is created on the fly and then imported. This is a potential feature since Python 3.3.

[strogdon]

I'm convinced that I need to call

importlib.invalidate_caches()

in order to get things to work properly. From the documentation there appears to be no harm in making the call. However, it seems that the call is not always needed and making the call unconditionally may introduce overhead. Therefore I propose to make the call conditionally as

src/sage/misc/cython.py

@@ -529 +529 @@
     try:
         sys.path.append(build_dir)
         return builtins.__import__(name)
+    except ModuleNotFoundError:
+        import importlib
+        importlib.invalidate_caches()
+        return builtins.__import__(name)
     finally:
         sys.path = oldpath

This fixed things for me. Comments are welcome.