File spyder-kernels-pr453.patch of Package python-spyder-kernels

Overview Repositories Revisions Requests Users Attributes Meta

File spyder-kernels-pr453.patch of Package python-spyder-kernels (Revision a46d410217fe7ce8afb42455db1d418e)

Currently displaying revision a46d410217fe7ce8afb42455db1d418e , Show latest

From ef612ee7df93bb462f05ebdfbce37a9c3b74bef1 Mon Sep 17 00:00:00 2001
From: Carlos Cordoba <ccordoba12@gmail.com>
Date: Wed, 5 Apr 2023 11:50:21 -0500
Subject: [PATCH 1/3] Move code that loads and saves Hdf5 and Dicom files from
 Spyder

That code was not working since the Spyder 5.0 release
---
 spyder_kernels/utils/iofuncs.py | 130 ++++++++++++++++++++++++--------
 1 file changed, 99 insertions(+), 31 deletions(-)

Index: spyder-kernels-2.4.4/spyder_kernels/utils/iofuncs.py
===================================================================
--- spyder-kernels-2.4.4.orig/spyder_kernels/utils/iofuncs.py
+++ spyder-kernels-2.4.4/spyder_kernels/utils/iofuncs.py
@@ -35,6 +35,8 @@ from spyder_kernels.utils.lazymodules im
     FakeObject, numpy as np, pandas as pd, PIL, scipy as sp)
 
 
+# ---- For Matlab files
+# -----------------------------------------------------------------------------
 class MatlabStruct(dict):
     """
     Matlab style struct, enhanced.
@@ -44,7 +46,7 @@ class MatlabStruct(dict):
 
     Examples
     ========
-    >>> from spyder.utils.iofuncs import MatlabStruct
+    >>> from spyder_kernels.utils.iofuncs import MatlabStruct
     >>> a = MatlabStruct()
     >>> a.b = 'spam'  # a["b"] == 'spam'
     >>> a.c["d"] = 'eggs'  # a.c.d == 'eggs'
@@ -169,6 +171,8 @@ def save_matlab(data, filename):
         return str(error)
 
 
+# ---- For arrays
+# -----------------------------------------------------------------------------
 def load_array(filename):
     if np.load is FakeObject:
         return None, ''
@@ -193,6 +197,8 @@ def __save_array(data, basename, index):
     return fname
 
 
+# ---- For PIL images
+# -----------------------------------------------------------------------------
 if sys.byteorder == 'little':
     _ENDIAN = '<'
 else:
@@ -237,6 +243,8 @@ def load_image(filename):
         return None, str(error)
 
 
+# ---- For misc formats
+# -----------------------------------------------------------------------------
 def load_pickle(filename):
     """Load a pickle file as a dictionary"""
     try:
@@ -264,6 +272,8 @@ def load_json(filename):
         return None, str(err)
 
 
+# ---- For Spydata files
+# -----------------------------------------------------------------------------
 def save_dictionary(data, filename):
     """Save dictionary in a single file .spydata file"""
     filename = osp.abspath(filename)
@@ -437,6 +447,89 @@ def load_dictionary(filename):
     return data, error_message
 
 
+# ---- For HDF5 files
+# -----------------------------------------------------------------------------
+def load_hdf5(filename):
+    """
+    Load an hdf5 file.
+
+    Notes
+    -----
+    - This is a fairly dumb implementation which reads the whole HDF5 file into
+      Spyder's variable explorer.  Since HDF5 files are designed for storing
+      very large data-sets, it may be much better to work directly with the
+      HDF5 objects, thus keeping the data on disk. Nonetheless, this gives
+      quick and dirty but convenient access to them.
+    - There is no support for creating files with compression, chunking etc,
+      although these can be read without problem.
+    - When reading an HDF5 file with sub-groups, groups in the file will
+      correspond to dictionaries with the same layout.
+    """
+    def get_group(group):
+        contents = {}
+        for name, obj in list(group.items()):
+            if isinstance(obj, h5py.Dataset):
+                contents[name] = np.array(obj)
+            elif isinstance(obj, h5py.Group):
+                # it is a group, so call self recursively
+                contents[name] = get_group(obj)
+            # other objects such as links are ignored
+        return contents
+
+    try:
+        import h5py
+
+        f = h5py.File(filename, 'r')
+        contents = get_group(f)
+        f.close()
+        return contents, None
+    except Exception as error:
+        return None, str(error)
+
+
+def save_hdf5(data, filename):
+    """
+    Save an hdf5 file.
+
+    Notes
+    -----
+    - All datatypes to be saved must be convertible to a numpy array, otherwise
+      an exception will be raised.
+    - Data attributes are currently ignored.
+    - When saving data after reading it with load_hdf5, dictionaries are not
+      turned into HDF5 groups.
+    """
+    try:
+        import h5py
+
+        f = h5py.File(filename, 'w')
+        for key, value in list(data.items()):
+            f[key] = np.array(value)
+        f.close()
+    except Exception as error:
+        return str(error)
+
+
+# ---- For DICOM files
+# -----------------------------------------------------------------------------
+def load_dicom(filename):
+    """Load a DICOM files."""
+    try:
+        from pydicom import dicomio
+
+        name = osp.splitext(osp.basename(filename))[0]
+        try:
+            data = dicomio.read_file(filename, force=True)
+        except TypeError:
+            data = dicomio.read_file(filename)
+        arr = data.pixel_array
+        return {name: arr}, None
+    except Exception as error:
+        return None, str(error)
+
+
+# ---- Class to group all IO functionality
+# -----------------------------------------------------------------------------
 class IOFunctions(object):
     def __init__(self):
         self.load_extensions = None
@@ -447,7 +540,7 @@ class IOFunctions(object):
         self.save_funcs = None
 
     def setup(self):
-        iofuncs = self.get_internal_funcs()+self.get_3rd_party_funcs()
+        iofuncs = self.get_internal_funcs()
         load_extensions = {}
         save_extensions = {}
         load_funcs = {}
@@ -455,6 +548,7 @@ class IOFunctions(object):
         load_filters = []
         save_filters = []
         load_ext = []
+
         for ext, name, loadfunc, savefunc in iofuncs:
             filter_str = to_text_string(name + " (*%s)" % ext)
             if loadfunc is not None:
@@ -466,9 +560,11 @@ class IOFunctions(object):
                 save_extensions[filter_str] = ext
                 save_filters.append(filter_str)
                 save_funcs[ext] = savefunc
-        load_filters.insert(0, to_text_string("Supported files"+" (*"+\
-                                              " *".join(load_ext)+")"))
+        load_filters.insert(
+            0, str("Supported files" + " (*" + " *".join(load_ext) + ")")
+        )
         load_filters.append(to_text_string("All files (*.*)"))
+
         self.load_filters = "\n".join(load_filters)
         self.save_filters = "\n".join(save_filters)
         self.load_funcs = load_funcs
@@ -478,35 +574,22 @@ class IOFunctions(object):
 
     def get_internal_funcs(self):
         return [
-                ('.spydata', "Spyder data files",
-                             load_dictionary, save_dictionary),
-                ('.npy', "NumPy arrays", load_array, None),
-                ('.npz', "NumPy zip arrays", load_array, None),
-                ('.mat', "Matlab files", load_matlab, save_matlab),
-                ('.csv', "CSV text files", 'import_wizard', None),
-                ('.txt', "Text files", 'import_wizard', None),
-                ('.jpg', "JPEG images", load_image, None),
-                ('.png', "PNG images", load_image, None),
-                ('.gif', "GIF images", load_image, None),
-                ('.tif', "TIFF images", load_image, None),
-                ('.pkl', "Pickle files", load_pickle, None),
-                ('.pickle', "Pickle files", load_pickle, None),
-                ('.json', "JSON files", load_json, None),
-                ]
-
-    def get_3rd_party_funcs(self):
-        other_funcs = []
-        try:
-            from spyder.otherplugins import get_spyderplugins_mods
-            for mod in get_spyderplugins_mods(io=True):
-                try:
-                    other_funcs.append((mod.FORMAT_EXT, mod.FORMAT_NAME,
-                                        mod.FORMAT_LOAD, mod.FORMAT_SAVE))
-                except AttributeError as error:
-                    print("%s: %s" % (mod, str(error)), file=sys.stderr)
-        except ImportError:
-            pass
-        return other_funcs
+            ('.spydata', "Spyder data files", load_dictionary, save_dictionary),
+            ('.npy', "NumPy arrays", load_array, None),
+            ('.npz', "NumPy zip arrays", load_array, None),
+            ('.mat', "Matlab files", load_matlab, save_matlab),
+            ('.csv', "CSV text files", 'import_wizard', None),
+            ('.txt', "Text files", 'import_wizard', None),
+            ('.jpg', "JPEG images", load_image, None),
+            ('.png', "PNG images", load_image, None),
+            ('.gif', "GIF images", load_image, None),
+            ('.tif', "TIFF images", load_image, None),
+            ('.pkl', "Pickle files", load_pickle, None),
+            ('.pickle', "Pickle files", load_pickle, None),
+            ('.json', "JSON files", load_json, None),
+            ('.h5', "HDF5 files", load_hdf5, save_hdf5),
+            ('.dcm', "DICOM images", load_dicom, None),
+        ]
 
     def save(self, data, filename):
         ext = osp.splitext(filename)[1].lower()
@@ -526,11 +609,8 @@ iofunctions = IOFunctions()
 iofunctions.setup()
 
 
-def save_auto(data, filename):
-    """Save data into filename, depending on file extension"""
-    pass
-
-
+# ---- Test
+# -----------------------------------------------------------------------------
 if __name__ == "__main__":
     import datetime
     testdict = {'d': 1, 'a': np.random.rand(10, 10), 'b': [1, 2]}
@@ -549,9 +629,9 @@ if __name__ == "__main__":
     import time
     t0 = time.time()
     save_dictionary(example, "test.spydata")
-    print(" Data saved in %.3f seconds" % (time.time()-t0))  # spyder: test-skip
+    print(" Data saved in %.3f seconds" % (time.time()-t0))
     t0 = time.time()
     example2, ok = load_dictionary("test.spydata")
     os.remove("test.spydata")
 
-    print("Data loaded in %.3f seconds" % (time.time()-t0))  # spyder: test-skip
+    print("Data loaded in %.3f seconds" % (time.time()-t0))
Index: spyder-kernels-2.4.4/.github/workflows/linux-pip-tests.yml
===================================================================
--- spyder-kernels-2.4.4.orig/.github/workflows/linux-pip-tests.yml
+++ spyder-kernels-2.4.4/.github/workflows/linux-pip-tests.yml
@@ -56,9 +56,9 @@ jobs:
       - name: Run tests
         shell: bash -l {0}
         run: |
-          xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -x -vv || \
-          xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -x -vv || \
-          xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -x -vv
+          xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -vv || \
+          xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -vv || \
+          xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -vv
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
Index: spyder-kernels-2.4.4/.github/workflows/linux-tests.yml
===================================================================
--- spyder-kernels-2.4.4.orig/.github/workflows/linux-tests.yml
+++ spyder-kernels-2.4.4/.github/workflows/linux-tests.yml
@@ -62,9 +62,9 @@ jobs:
       - name: Run tests
         shell: bash -l {0}
         run: |
-          xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -x -vv || \
-          xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -x -vv || \
-          xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -x -vv
+          xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -vv || \
+          xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -vv || \
+          xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -vv
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
Index: spyder-kernels-2.4.4/requirements/tests.txt
===================================================================
--- spyder-kernels-2.4.4.orig/requirements/tests.txt
+++ spyder-kernels-2.4.4/requirements/tests.txt
@@ -11,3 +11,5 @@ scipy
 xarray
 pillow
 django
+h5py
+pydicom
Index: spyder-kernels-2.4.4/setup.py
===================================================================
--- spyder-kernels-2.4.4.orig/setup.py
+++ spyder-kernels-2.4.4/setup.py
@@ -64,6 +64,8 @@ TEST_REQUIREMENTS = [
     'xarray',
     'pillow',
     'django',
+    'h5py',
+    'pydicom'
 ]
 
 setup(
Index: spyder-kernels-2.4.4/spyder_kernels/utils/tests/test_iofuncs.py
===================================================================
--- spyder-kernels-2.4.4.orig/spyder_kernels/utils/tests/test_iofuncs.py
+++ spyder-kernels-2.4.4/spyder_kernels/utils/tests/test_iofuncs.py
@@ -16,6 +16,7 @@ import os
 import copy
 
 # Third party imports
+from PIL import ImageFile
 import pytest
 import numpy as np
 
@@ -25,8 +26,9 @@ from spyder_kernels.py3compat import is_
 
 
 # Full path to this file's parent directory for loading data
-LOCATION = os.path.realpath(os.path.join(os.getcwd(),
-                                         os.path.dirname(__file__)))
+LOCATION = os.path.realpath(
+    os.path.join(os.getcwd(), os.path.dirname(__file__))
+)
 
 
 # =============================================================================
@@ -340,5 +342,25 @@ def test_spydata_export(input_namespace,
                 pass
 
 
+def test_save_load_hdf5_files():
+    """Simple test to check that we can save and load HDF5 files."""
+    data = {'a' : [1, 2, 3, 4], 'b' : 4.5}
+    iofuncs.save_hdf5(data, "test.h5")
+
+    expected = ({'a': np.array([1, 2, 3, 4]), 'b': np.array(4.5)}, None)
+    assert repr(iofuncs.load_hdf5("test.h5")) == repr(expected)
+
+
+def test_load_dicom_files():
+    """Check that we can load DICOM files."""
+    # This test pass locally but we need to set the variable below for it to
+    # pass on CIs.
+    # See https://stackoverflow.com/a/47958486/438386 for context.
+    ImageFile.LOAD_TRUNCATED_IMAGES = True
+
+    data = iofuncs.load_dicom(os.path.join(LOCATION, 'data.dcm'))
+    assert data[0]['data'].shape == (512, 512)
+
+
 if __name__ == "__main__":
     pytest.main()

Places

File spyder-kernels-pr453.patch of Package python-spyder-kernels (Revision a46d410217fe7ce8afb42455db1d418e)

Places