Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
devel:languages:python:numeric
python-spyder-kernels
spyder-kernels-pr453.patch
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File spyder-kernels-pr453.patch of Package python-spyder-kernels
From ef612ee7df93bb462f05ebdfbce37a9c3b74bef1 Mon Sep 17 00:00:00 2001 From: Carlos Cordoba <ccordoba12@gmail.com> Date: Wed, 5 Apr 2023 11:50:21 -0500 Subject: [PATCH 1/3] Move code that loads and saves Hdf5 and Dicom files from Spyder That code was not working since the Spyder 5.0 release --- spyder_kernels/utils/iofuncs.py | 130 ++++++++++++++++++++++++-------- 1 file changed, 99 insertions(+), 31 deletions(-) Index: spyder-kernels-2.4.4/spyder_kernels/utils/iofuncs.py =================================================================== --- spyder-kernels-2.4.4.orig/spyder_kernels/utils/iofuncs.py +++ spyder-kernels-2.4.4/spyder_kernels/utils/iofuncs.py @@ -35,6 +35,8 @@ from spyder_kernels.utils.lazymodules im FakeObject, numpy as np, pandas as pd, PIL, scipy as sp) +# ---- For Matlab files +# ----------------------------------------------------------------------------- class MatlabStruct(dict): """ Matlab style struct, enhanced. @@ -44,7 +46,7 @@ class MatlabStruct(dict): Examples ======== - >>> from spyder.utils.iofuncs import MatlabStruct + >>> from spyder_kernels.utils.iofuncs import MatlabStruct >>> a = MatlabStruct() >>> a.b = 'spam' # a["b"] == 'spam' >>> a.c["d"] = 'eggs' # a.c.d == 'eggs' @@ -169,6 +171,8 @@ def save_matlab(data, filename): return str(error) +# ---- For arrays +# ----------------------------------------------------------------------------- def load_array(filename): if np.load is FakeObject: return None, '' @@ -193,6 +197,8 @@ def __save_array(data, basename, index): return fname +# ---- For PIL images +# ----------------------------------------------------------------------------- if sys.byteorder == 'little': _ENDIAN = '<' else: @@ -237,6 +243,8 @@ def load_image(filename): return None, str(error) +# ---- For misc formats +# ----------------------------------------------------------------------------- def load_pickle(filename): """Load a pickle file as a dictionary""" try: @@ -264,6 +272,8 @@ def load_json(filename): return None, str(err) +# ---- For Spydata files +# ----------------------------------------------------------------------------- def save_dictionary(data, filename): """Save dictionary in a single file .spydata file""" filename = osp.abspath(filename) @@ -437,6 +447,89 @@ def load_dictionary(filename): return data, error_message +# ---- For HDF5 files +# ----------------------------------------------------------------------------- +def load_hdf5(filename): + """ + Load an hdf5 file. + + Notes + ----- + - This is a fairly dumb implementation which reads the whole HDF5 file into + Spyder's variable explorer. Since HDF5 files are designed for storing + very large data-sets, it may be much better to work directly with the + HDF5 objects, thus keeping the data on disk. Nonetheless, this gives + quick and dirty but convenient access to them. + - There is no support for creating files with compression, chunking etc, + although these can be read without problem. + - When reading an HDF5 file with sub-groups, groups in the file will + correspond to dictionaries with the same layout. + """ + def get_group(group): + contents = {} + for name, obj in list(group.items()): + if isinstance(obj, h5py.Dataset): + contents[name] = np.array(obj) + elif isinstance(obj, h5py.Group): + # it is a group, so call self recursively + contents[name] = get_group(obj) + # other objects such as links are ignored + return contents + + try: + import h5py + + f = h5py.File(filename, 'r') + contents = get_group(f) + f.close() + return contents, None + except Exception as error: + return None, str(error) + + +def save_hdf5(data, filename): + """ + Save an hdf5 file. + + Notes + ----- + - All datatypes to be saved must be convertible to a numpy array, otherwise + an exception will be raised. + - Data attributes are currently ignored. + - When saving data after reading it with load_hdf5, dictionaries are not + turned into HDF5 groups. + """ + try: + import h5py + + f = h5py.File(filename, 'w') + for key, value in list(data.items()): + f[key] = np.array(value) + f.close() + except Exception as error: + return str(error) + + +# ---- For DICOM files +# ----------------------------------------------------------------------------- +def load_dicom(filename): + """Load a DICOM files.""" + try: + from pydicom import dicomio + + name = osp.splitext(osp.basename(filename))[0] + try: + data = dicomio.read_file(filename, force=True) + except TypeError: + data = dicomio.read_file(filename) + arr = data.pixel_array + return {name: arr}, None + except Exception as error: + return None, str(error) + + +# ---- Class to group all IO functionality +# ----------------------------------------------------------------------------- class IOFunctions(object): def __init__(self): self.load_extensions = None @@ -447,7 +540,7 @@ class IOFunctions(object): self.save_funcs = None def setup(self): - iofuncs = self.get_internal_funcs()+self.get_3rd_party_funcs() + iofuncs = self.get_internal_funcs() load_extensions = {} save_extensions = {} load_funcs = {} @@ -455,6 +548,7 @@ class IOFunctions(object): load_filters = [] save_filters = [] load_ext = [] + for ext, name, loadfunc, savefunc in iofuncs: filter_str = to_text_string(name + " (*%s)" % ext) if loadfunc is not None: @@ -466,9 +560,11 @@ class IOFunctions(object): save_extensions[filter_str] = ext save_filters.append(filter_str) save_funcs[ext] = savefunc - load_filters.insert(0, to_text_string("Supported files"+" (*"+\ - " *".join(load_ext)+")")) + load_filters.insert( + 0, str("Supported files" + " (*" + " *".join(load_ext) + ")") + ) load_filters.append(to_text_string("All files (*.*)")) + self.load_filters = "\n".join(load_filters) self.save_filters = "\n".join(save_filters) self.load_funcs = load_funcs @@ -478,35 +574,22 @@ class IOFunctions(object): def get_internal_funcs(self): return [ - ('.spydata', "Spyder data files", - load_dictionary, save_dictionary), - ('.npy', "NumPy arrays", load_array, None), - ('.npz', "NumPy zip arrays", load_array, None), - ('.mat', "Matlab files", load_matlab, save_matlab), - ('.csv', "CSV text files", 'import_wizard', None), - ('.txt', "Text files", 'import_wizard', None), - ('.jpg', "JPEG images", load_image, None), - ('.png', "PNG images", load_image, None), - ('.gif', "GIF images", load_image, None), - ('.tif', "TIFF images", load_image, None), - ('.pkl', "Pickle files", load_pickle, None), - ('.pickle', "Pickle files", load_pickle, None), - ('.json', "JSON files", load_json, None), - ] - - def get_3rd_party_funcs(self): - other_funcs = [] - try: - from spyder.otherplugins import get_spyderplugins_mods - for mod in get_spyderplugins_mods(io=True): - try: - other_funcs.append((mod.FORMAT_EXT, mod.FORMAT_NAME, - mod.FORMAT_LOAD, mod.FORMAT_SAVE)) - except AttributeError as error: - print("%s: %s" % (mod, str(error)), file=sys.stderr) - except ImportError: - pass - return other_funcs + ('.spydata', "Spyder data files", load_dictionary, save_dictionary), + ('.npy', "NumPy arrays", load_array, None), + ('.npz', "NumPy zip arrays", load_array, None), + ('.mat', "Matlab files", load_matlab, save_matlab), + ('.csv', "CSV text files", 'import_wizard', None), + ('.txt', "Text files", 'import_wizard', None), + ('.jpg', "JPEG images", load_image, None), + ('.png', "PNG images", load_image, None), + ('.gif', "GIF images", load_image, None), + ('.tif', "TIFF images", load_image, None), + ('.pkl', "Pickle files", load_pickle, None), + ('.pickle', "Pickle files", load_pickle, None), + ('.json', "JSON files", load_json, None), + ('.h5', "HDF5 files", load_hdf5, save_hdf5), + ('.dcm', "DICOM images", load_dicom, None), + ] def save(self, data, filename): ext = osp.splitext(filename)[1].lower() @@ -526,11 +609,8 @@ iofunctions = IOFunctions() iofunctions.setup() -def save_auto(data, filename): - """Save data into filename, depending on file extension""" - pass - - +# ---- Test +# ----------------------------------------------------------------------------- if __name__ == "__main__": import datetime testdict = {'d': 1, 'a': np.random.rand(10, 10), 'b': [1, 2]} @@ -549,9 +629,9 @@ if __name__ == "__main__": import time t0 = time.time() save_dictionary(example, "test.spydata") - print(" Data saved in %.3f seconds" % (time.time()-t0)) # spyder: test-skip + print(" Data saved in %.3f seconds" % (time.time()-t0)) t0 = time.time() example2, ok = load_dictionary("test.spydata") os.remove("test.spydata") - print("Data loaded in %.3f seconds" % (time.time()-t0)) # spyder: test-skip + print("Data loaded in %.3f seconds" % (time.time()-t0)) Index: spyder-kernels-2.4.4/.github/workflows/linux-pip-tests.yml =================================================================== --- spyder-kernels-2.4.4.orig/.github/workflows/linux-pip-tests.yml +++ spyder-kernels-2.4.4/.github/workflows/linux-pip-tests.yml @@ -56,9 +56,9 @@ jobs: - name: Run tests shell: bash -l {0} run: | - xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -x -vv || \ - xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -x -vv || \ - xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -x -vv + xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -vv || \ + xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -vv || \ + xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -vv - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: Index: spyder-kernels-2.4.4/.github/workflows/linux-tests.yml =================================================================== --- spyder-kernels-2.4.4.orig/.github/workflows/linux-tests.yml +++ spyder-kernels-2.4.4/.github/workflows/linux-tests.yml @@ -62,9 +62,9 @@ jobs: - name: Run tests shell: bash -l {0} run: | - xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -x -vv || \ - xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -x -vv || \ - xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -x -vv + xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -vv || \ + xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -vv || \ + xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -vv - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: Index: spyder-kernels-2.4.4/requirements/tests.txt =================================================================== --- spyder-kernels-2.4.4.orig/requirements/tests.txt +++ spyder-kernels-2.4.4/requirements/tests.txt @@ -11,3 +11,5 @@ scipy xarray pillow django +h5py +pydicom Index: spyder-kernels-2.4.4/setup.py =================================================================== --- spyder-kernels-2.4.4.orig/setup.py +++ spyder-kernels-2.4.4/setup.py @@ -64,6 +64,8 @@ TEST_REQUIREMENTS = [ 'xarray', 'pillow', 'django', + 'h5py', + 'pydicom' ] setup( Index: spyder-kernels-2.4.4/spyder_kernels/utils/tests/test_iofuncs.py =================================================================== --- spyder-kernels-2.4.4.orig/spyder_kernels/utils/tests/test_iofuncs.py +++ spyder-kernels-2.4.4/spyder_kernels/utils/tests/test_iofuncs.py @@ -16,6 +16,7 @@ import os import copy # Third party imports +from PIL import ImageFile import pytest import numpy as np @@ -25,8 +26,9 @@ from spyder_kernels.py3compat import is_ # Full path to this file's parent directory for loading data -LOCATION = os.path.realpath(os.path.join(os.getcwd(), - os.path.dirname(__file__))) +LOCATION = os.path.realpath( + os.path.join(os.getcwd(), os.path.dirname(__file__)) +) # ============================================================================= @@ -340,5 +342,25 @@ def test_spydata_export(input_namespace, pass +def test_save_load_hdf5_files(): + """Simple test to check that we can save and load HDF5 files.""" + data = {'a' : [1, 2, 3, 4], 'b' : 4.5} + iofuncs.save_hdf5(data, "test.h5") + + expected = ({'a': np.array([1, 2, 3, 4]), 'b': np.array(4.5)}, None) + assert repr(iofuncs.load_hdf5("test.h5")) == repr(expected) + + +def test_load_dicom_files(): + """Check that we can load DICOM files.""" + # This test pass locally but we need to set the variable below for it to + # pass on CIs. + # See https://stackoverflow.com/a/47958486/438386 for context. + ImageFile.LOAD_TRUNCATED_IMAGES = True + + data = iofuncs.load_dicom(os.path.join(LOCATION, 'data.dcm')) + assert data[0]['data'].shape == (512, 512) + + if __name__ == "__main__": pytest.main()
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor
