File kst_netcdf4_compat.patch of Package kst
From c2b585fbd23e1e7de8f7dba457ca632bb07708da Mon Sep 17 00:00:00 2001
From: Sasha Rahlin <arahlin@uchicago.edu>
Date: Mon, 23 Dec 2024 12:47:50 +1300
Subject: [PATCH] Update netCDF datasource plugin to work with modern netCDF4
C++ bindings
---
cmake/modules/FindNetcdf.cmake | 10 +-
src/datasources/netcdf/netcdfplugin.cpp | 6 +-
src/datasources/netcdf/netcdfsource.cpp | 282 +++++++++++-------------
src/datasources/netcdf/netcdfsource.h | 8 +-
4 files changed, 132 insertions(+), 174 deletions(-)
diff --git a/cmake/modules/FindNetcdf.cmake b/cmake/modules/FindNetcdf.cmake
index c87f40074..88236abc5 100644
--- a/cmake/modules/FindNetcdf.cmake
+++ b/cmake/modules/FindNetcdf.cmake
@@ -17,11 +17,11 @@ if(NOT NETCDF_INCLUDEDIR)
if(NOT kst_cross)
include(FindPkgConfig)
- pkg_check_modules(NETCDF QUIET netcdf)
+ pkg_check_modules(NETCDF netcdf netcdf-cxx4)
endif()
if(NETCDF_INCLUDEDIR AND NETCDF_LIBRARIES)
- FIND_LIBRARY(NETCDF_LIBRARY_CPP netcdf_c++
+ FIND_LIBRARY(NETCDF_LIBRARY_CPP netcdf_c++4
HINTS ${NETCDF_LIBRARY_DIRS})
set(NETCDF_LIBRARY_C -L${NETCDF_LIBRARY_DIRS} ${NETCDF_LIBRARIES} CACHE STRING "" FORCE)
else()
@@ -46,9 +46,9 @@ else()
find_netcdf_lib(netcdf_c netcdf)
find_netcdf_lib(netcdf_c_debug netcdfd)
- find_netcdf_lib(netcdf_cpp netcdf_c++)
- find_netcdf_lib(netcdf_cpp_debug netcdf_c++d)
-
+ find_netcdf_lib(netcdf_cpp netcdf_c++4)
+ find_netcdf_lib(netcdf_cpp_debug netcdf_c++4d)
+
if(netcdf_c AND netcdf_c_debug)
set(NETCDF_LIBRARY_C optimized ${netcdf_c} debug ${netcdf_c_debug} CACHE STRING "" FORCE)
endif()
diff --git a/src/datasources/netcdf/netcdfplugin.cpp b/src/datasources/netcdf/netcdfplugin.cpp
index 01cdb0f6b..74bf46736 100644
--- a/src/datasources/netcdf/netcdfplugin.cpp
+++ b/src/datasources/netcdf/netcdfplugin.cpp
@@ -126,13 +126,11 @@ int NetCdfPlugin::understands(QSettings *cfg, const QString& filename) const
return 0;
}
- NcFile *ncfile = new NcFile(filename.toUtf8().data());
- if (ncfile->is_valid()) {
+ netCDF::NcFile ncfile(filename.toUtf8().data(), netCDF::NcFile::read);
+ if (!ncfile.isNull()) {
KST_DBG qDebug() << filename << " looks like netCDF !" << endl;
- delete ncfile;
return 80;
} else {
- delete ncfile;
return 0;
}
}
diff --git a/src/datasources/netcdf/netcdfsource.cpp b/src/datasources/netcdf/netcdfsource.cpp
index 4a5a466b2..65ccbc124 100644
--- a/src/datasources/netcdf/netcdfsource.cpp
+++ b/src/datasources/netcdf/netcdfsource.cpp
@@ -38,6 +38,7 @@
using namespace Kst;
+using namespace netCDF;
static const QString netCdfTypeString = "netCDF Files";
@@ -192,50 +193,72 @@ bool DataInterfaceNetCdfVector::isValid(const QString& field) const
QMap<QString, double> DataInterfaceNetCdfVector::metaScalars(const QString& field)
{
- NcVar *var = 0;
+ NcVar var;
if (field != "INDEX") {
- var = netcdf._ncfile->get_var((NcToken) field.toLatin1().constData());
+ var = netcdf._ncfile->getVar(field.toLatin1().constData());
}
- if (!var) {
+ if (var.isNull()) {
NETCDF_DBG qDebug() << "Queried field " << field << " which can't be read" << endl;
return QMap<QString, double>();
}
QMap<QString, double> fieldScalars;
- fieldScalars["NbAttributes"] = var->num_atts();
- for (int i=0; i<var->num_atts(); ++i) {
- NcAtt *att = var->get_att(i);
+ fieldScalars["NbAttributes"] = var.getAttCount();
+ for (auto att: var.getAtts()) {
+ size_t n = att.second.getAttLength();
+ std::vector<double> data(n);
// Only handle char attributes as fieldStrings, the others as fieldScalars
- if (att->type() == ncByte || att->type() == ncShort || att->type() == ncInt
- || att->type() == ncLong || att->type() == ncFloat || att->type() == ncDouble) {
- // Some attributes may have multiple values => load the first as is, and for the others
- // add a -2, -3, etc... suffix as obviously we can have only one value per scalar.
- // Do it in two steps to avoid a test in the loop while keeping a "clean" name for the first one
- fieldScalars[QString(att->name())] = att->values()->as_double(0);
- for (int j=1; j<att->values()->num(); ++j) {
- fieldScalars[QString(att->name()) + QString("-") + QString::number(j+1)] = att->values()->as_double(j);
- }
+ switch (att.second.getType().getTypeClass()) {
+ case NC_BYTE:
+ case NC_UBYTE:
+ case NC_SHORT:
+ case NC_USHORT:
+ case NC_INT:
+ case NC_UINT:
+ case NC_INT64:
+ case NC_UINT64:
+ case NC_FLOAT:
+ case NC_DOUBLE:
+ att.second.getValues((double *)&data[0]);
+ break;
+ default:
+ break;
}
+ // Some attributes may have multiple values => load the first as is, and for the others
+ // add a -2, -3, etc... suffix as obviously we can have only one value per scalar.
+ // Do it in two steps to avoid a test in the loop while keeping a "clean" name for the first one
+ QString name(att.first.c_str());
+ fieldScalars[name] = data[0];
+ for (size_t j=1; j<n; j++)
+ fieldScalars[name + QString("-") + QString::number(j+1)] = data[j];
}
return fieldScalars;
}
QMap<QString, QString> DataInterfaceNetCdfVector::metaStrings(const QString& field)
{
- NcVar *var = 0;
+ NcVar var;
if (field != "INDEX") {
- var = netcdf._ncfile->get_var(field.toLatin1().constData());
+ var = netcdf._ncfile->getVar(field.toLatin1().constData());
}
- if (!var) {
+ if (var.isNull()) {
NETCDF_DBG qDebug() << "Queried field " << field << " which can't be read" << endl;
return QMap<QString, QString>();
}
QMap<QString, QString> fieldStrings;
QString tmpString;
- for (int i=0; i<var->num_atts(); ++i) {
- NcAtt *att = var->get_att(i);
+ for (auto att: var.getAtts()) {
// Only handle char/unspecified attributes as fieldStrings, the others as fieldScalars
- if (att->type() == ncChar || att->type() == ncNoType) {
- fieldStrings[att->name()] = QString(att->values()->as_string(0));
+ switch (att.second.getType().getTypeClass()) {
+ case NC_CHAR:
+ case NC_STRING:
+ {
+ std::string s;
+ att.second.getValues(s);
+ fieldStrings[QString(att.first.c_str())] = QString(s.c_str());
+ }
+ break;
+ default:
+ break;
}
// qDebug() << att->name() << ": " << att->values()->num() << endl;
}
@@ -282,20 +305,19 @@ const DataMatrix::DataInfo DataInterfaceNetCdfMatrix::dataInfo(const QString& ma
return DataMatrix::DataInfo();
}
- QByteArray bytes = matrix.toLatin1();
- NcVar *var = netcdf._ncfile->get_var(bytes.constData()); // var is owned by _ncfile
- if (!var) {
+ NcVar var = netcdf._ncfile->getVar(matrix.toLatin1().constData()); // var is owned by _ncfile
+ if (var.isNull()) {
return DataMatrix::DataInfo();
}
- if (var->num_dims() != 2) {
+ if (var.getDimCount() != 2) {
return DataMatrix::DataInfo();
}
DataMatrix::DataInfo info;
// TODO is this right?
- info.xSize = var->get_dim(0)->size();
- info.ySize = var->get_dim(1)->size();
+ info.xSize = var.getDim(0).getSize();
+ info.ySize = var.getDim(1).getSize();
return info;
}
@@ -326,7 +348,6 @@ bool DataInterfaceNetCdfMatrix::isValid(const QString& field) const {
NetcdfSource::NetcdfSource(Kst::ObjectStore *store, QSettings *cfg, const QString& filename, const QString& type, const QDomElement &element) :
Kst::DataSource(store, cfg, filename, type),
_ncfile(0L),
- _ncErr(NcError::silent_nonfatal),
is(new DataInterfaceNetCdfScalar(*this)),
it(new DataInterfaceNetCdfString(*this)),
iv(new DataInterfaceNetCdfVector(*this)),
@@ -367,8 +388,8 @@ void NetcdfSource::reset() {
bool NetcdfSource::initFile() {
- _ncfile = new NcFile(_filename.toUtf8().data(), NcFile::ReadOnly);
- if (!_ncfile->is_valid()) {
+ _ncfile = new NcFile(_filename.toUtf8().data(), NcFile::read);
+ if (_ncfile->isNull()) {
qDebug() << _filename << ": failed to open in initFile()" << endl;
return false;
}
@@ -377,43 +398,38 @@ bool NetcdfSource::initFile() {
_fieldList.clear();
_fieldList += "INDEX";
- int nb_vars = _ncfile->num_vars();
- NETCDF_DBG qDebug() << nb_vars << " vars found in total" << endl;
+ NETCDF_DBG qDebug() << _ncfile->getVarCount() << " vars found in total" << endl;
_maxFrameCount = 0;
- for (int i = 0; i < nb_vars; i++) {
- NcVar *var = _ncfile->get_var(i);
- if (!var) {
+ for (auto var: _ncfile->getVars()) {
+ if (var.second.isNull()) {
continue;
}
- if (var->num_dims() == 0) {
- _scalarList += var->name();
- } else if (var->num_dims() == 1) {
- _fieldList += var->name();
- int fc = var->num_vals() / var->rec_size();
+ if (var.second.getDimCount() == 0) {
+ _scalarList += var.first.c_str();
+ } else if (var.second.getDimCount() == 1) {
+ _fieldList += var.first.c_str();
+ int fc = var.second.getDim(0).getSize();
_maxFrameCount = qMax(_maxFrameCount, fc);
- _frameCounts[var->name()] = fc;
- } else if (var->num_dims() == 2) {
- _matrixList += var->name();
+ _frameCounts[QString(var.first.c_str())] = fc;
+ } else if (var.second.getDimCount() == 2) {
+ _matrixList += var.first.c_str();
}
}
// Get strings
- int globalAttributesNb = _ncfile->num_atts();
- for (int i = 0; i < globalAttributesNb; ++i) {
+ for (auto att: _ncfile->getAtts()) {
// Get only first value, should be enough for a start especially as strings are complete
- NcAtt *att = _ncfile->get_att(i);
- if (att) {
- QString attrName = QString(att->name());
- char *attString = att->as_string(0);
- QString attrValue = QString(att->as_string(0));
- delete[] attString;
+ if (!att.second.isNull()) {
+ QString attrName = QString(att.first.c_str());
+ std::string s;
+ att.second.getValues(s);
+ QString attrValue = QString(s.c_str());
//TODO port
//KstString *ms = new KstString(KstObjectTag(attrName, tag()), this, attrValue);
_strings[attrName] = attrValue;
}
- delete att;
}
setUpdateType(Timer);
@@ -437,16 +453,17 @@ Kst::Object::UpdateType NetcdfSource::internalDataSourceUpdate() {
bool updated = false;
/* Update member variables _ncfile, _maxFrameCount, and _frameCounts
and indicate that an update is needed */
- int nb_vars = _ncfile->num_vars();
- for (int j = 0; j < nb_vars; j++) {
- NcVar *var = _ncfile->get_var(j);
- if (!var) {
+ for (auto var: _ncfile->getVars()) {
+ if (var.second.isNull()) {
continue;
}
- int fc = var->num_vals() / var->rec_size();
- _maxFrameCount = qMax(_maxFrameCount, fc);
- updated = updated || (_frameCounts[var->name()] != fc);
- _frameCounts[var->name()] = fc;
+ if (var.second.getDimCount() == 1) {
+ int fc = var.second.getDim(0).getSize();
+ _maxFrameCount = qMax(_maxFrameCount, fc);
+ QString name(var.first.c_str());
+ updated = updated || (_frameCounts[name] != fc);
+ _frameCounts[name] = fc;
+ }
}
return updated ? Object::Updated : Object::NoChange;
}
@@ -455,10 +472,9 @@ Kst::Object::UpdateType NetcdfSource::internalDataSourceUpdate() {
int NetcdfSource::readScalar(double *v, const QString& field)
{
// TODO error handling
- QByteArray bytes = field.toLatin1();
- NcVar *var = _ncfile->get_var(bytes.constData()); // var is owned by _ncfile
- if (var) {
- var->get(v);
+ NcVar var = _ncfile->getVar(field.toLatin1().constData()); // var is owned by _ncfile
+ if (!var.isNull()) {
+ var.getVar(v);
return 1;
}
return 0;
@@ -467,20 +483,17 @@ int NetcdfSource::readScalar(double *v, const QString& field)
int NetcdfSource::readString(QString *stringValue, const QString& stringName)
{
// TODO more error handling?
- NcAtt *att = _ncfile->get_att((NcToken) stringName.toLatin1().data());
- if (att) {
- *stringValue = QString(att->as_string(0));
- delete att;
+ NcGroupAtt att = _ncfile->getAtt(stringName.toLatin1().constData());
+ if (!att.isNull()) {
+ std::string s;
+ att.getValues(s);
+ *stringValue = QString(s.c_str());
return 1;
}
return 0;
}
int NetcdfSource::readField(double *v, const QString& field, int s, int n) {
- NcType dataType = ncNoType; /* netCDF data type */
- /* Values for one record */
- NcValues *record = 0;// = new NcValues(dataType,numFrameVals);
-
NETCDF_DBG qDebug() << "Entering NetcdfSource::readField with params: " << field << ", from " << s << " for " << n << " frames" << endl;
/* For INDEX field */
@@ -496,24 +509,24 @@ int NetcdfSource::readField(double *v, const QString& field, int s, int n) {
}
/* For a variable from the netCDF file */
- QByteArray bytes = field.toLatin1();
- NcVar *var = _ncfile->get_var(bytes.constData()); // var is owned by _ncfile
- if (!var) {
+ NcVar var = _ncfile->getVar(field.toLatin1().constData()); // var is owned by _ncfile
+ if (var.isNull()) {
NETCDF_DBG qDebug() << "Queried field " << field << " which can't be read" << endl;
return -1;
}
- dataType = var->type();
-
- if (s >= var->num_vals() / var->rec_size()) {
+ if (s >= (int) var.getDim(0).getSize()) {
return 0;
}
bool oneSample = n < 0;
- int recSize = var->rec_size();
double add_offset = 1.0, scale_factor = 1.0;
- switch (dataType) {
- case ncShort:
+ std::vector<size_t> sv(1);
+ sv[0] = s;
+ std::vector<size_t> nv(1);
+ nv[0] = n;
+ switch (var.getType().getTypeClass()) {
+ case NC_SHORT:
{
// Check for special attributes add_offset and scale_factor indicating the use of the convention described in
// <http://www.unidata.ucar.edu/software/netcdf/docs/netcdf/Attribute-Conventions.html>
@@ -524,83 +537,37 @@ int NetcdfSource::readField(double *v, const QString& field, int s, int n) {
scale_factor = iv->metaScalars(field)["scale_factor"];
}
if (oneSample) {
- record = var->get_rec(s);
- v[0] = packed ? record->as_short(0)*scale_factor+add_offset : record->as_short(0);
- delete record;
+ short record;
+ var.getVar(sv, &record);
+ v[0] = packed ? record*scale_factor+add_offset : record;
} else {
- for (int i = 0; i < n; i++) {
- record = var->get_rec(i+s);
- if (packed) {
- for (int j = 0; j < recSize; j++) {
- v[i*recSize + j] = record->as_short(j)*scale_factor+add_offset;
- }
- }
- else {
- for (int j = 0; j < recSize; j++) {
- v[i*recSize + j] = record->as_short(j);
- }
- }
- delete record;
+ std::vector<short> record(n);
+ var.getVar(sv, nv, (short *)&record[0]);
+ for (int i = 0; i < n; i++) {
+ if (packed) {
+ v[i] = record[i]*scale_factor+add_offset;
+ } else {
+ v[i] = record[i];
+ }
}
}
}
break;
- case ncInt:
+ case NC_INT:
+ case NC_UINT:
+ case NC_INT64:
+ case NC_UINT64:
+ case NC_FLOAT:
+ case NC_DOUBLE:
{
if (oneSample) {
- record = var->get_rec(s);
- v[0] = record->as_int(0);
- delete record;
+ var.getVar(sv, v);
} else {
- for (int i = 0; i < n; i++) {
- record = var->get_rec(i+s);
- NETCDF_DBG qDebug() << "Read record " << i+s << endl;
- for (int j = 0; j < recSize; j++) {
- v[i*recSize + j] = record->as_int(j);
- }
- delete record;
- }
+ var.getVar(sv, nv, v);
}
}
break;
-
- case ncFloat:
- {
- if (oneSample) {
- record = var->get_rec(s);
- v[0] = record->as_float(0);
- delete record;
- } else {
- for (int i = 0; i < n; i++) {
- record = var->get_rec(i+s);
- for (int j = 0; j < recSize; j++) {
- v[i*recSize + j] = record->as_float(j);
- }
- delete record;
- }
- }
- }
- break;
-
- case ncDouble:
- {
- if (oneSample) {
- record = var->get_rec(s);
- v[0] = record->as_double(0);
- delete record;
- } else {
- for (int i = 0; i < n; i++) {
- record = var->get_rec(i+s);
- for (int j = 0; j < recSize; j++) {
- v[i*recSize + j] = record->as_double(j);
- }
- delete record;
- }
- }
- }
- break;
-
default:
NETCDF_DBG qDebug() << field << ": wrong datatype for kst, no values read" << endl;
return -1;
@@ -610,7 +577,7 @@ int NetcdfSource::readField(double *v, const QString& field, int s, int n) {
NETCDF_DBG qDebug() << "Finished reading " << field << endl;
- return oneSample ? 1 : n * recSize;
+ return oneSample ? 1 : n;
}
@@ -620,19 +587,17 @@ int NetcdfSource::readField(double *v, const QString& field, int s, int n) {
int NetcdfSource::readMatrix(double *v, const QString& field)
{
/* For a variable from the netCDF file */
- QByteArray bytes = field.toLatin1();
- NcVar *var = _ncfile->get_var(bytes.constData()); // var is owned by _ncfile
- if (!var) {
+ NcVar var = _ncfile->getVar(field.toLatin1().constData()); // var is owned by _ncfile
+ if (var.isNull()) {
NETCDF_DBG qDebug() << "Queried field " << field << " which can't be read" << endl;
return -1;
}
- int xSize = var->get_dim(0)->size();
- int ySize = var->get_dim(1)->size();
+ int xSize = var.getDim(0).getSize();
+ int ySize = var.getDim(1).getSize();
- var->get(v, xSize, ySize);
+ var.getVar(v);
-
return xSize * ySize;
}
@@ -647,13 +612,12 @@ int NetcdfSource::samplesPerFrame(const QString& field) {
if (field.toLower() == "index") {
return 1;
}
- QByteArray bytes = field.toLatin1();
- NcVar *var = _ncfile->get_var(bytes.constData());
- if (!var) {
+ NcVar var = _ncfile->getVar(field.toLatin1().constData());
+ if (var.isNull()) {
NETCDF_DBG qDebug() << "Queried field " << field << " which can't be read" << endl;
return 0;
}
- return var->rec_size();
+ return 1;
}
diff --git a/src/datasources/netcdf/netcdfsource.h b/src/datasources/netcdf/netcdfsource.h
index 7bd7dca77..da77a7454 100644
--- a/src/datasources/netcdf/netcdfsource.h
+++ b/src/datasources/netcdf/netcdfsource.h
@@ -22,8 +22,7 @@
#include "datasource.h"
#include "dataplugin.h"
-#include <netcdf.h>
-#include <netcdfcpp.h>
+#include <netcdf>
class DataInterfaceNetCdfScalar;
@@ -71,10 +70,7 @@ class NetcdfSource : public Kst::DataSource {
QMap<QString, int> _frameCounts;
int _maxFrameCount;
- NcFile *_ncfile;
-
- // we must hold an NcError to overwrite the exit-on-error behaviour of netCDF
- NcError _ncErr;
+ netCDF::NcFile *_ncfile;
QMap<QString, QString> _strings;
--
GitLab
