File 0001-enable-sql-metadata-query.patch of Package openstack-ceilometer
From 02d93a52fa68d2ed75f0fbb1cb00f9aebaa5069c Mon Sep 17 00:00:00 2001
From: Gordon Chung <chungg@ca.ibm.com>
Date: Wed, 2 Oct 2013 15:45:26 -0400
Subject: [PATCH] enable sql metadata query
explode metadata key/values to their own tables/rows (based on type).
build a key string using dot notation similar to other nosql db
and filter based on that.
Blueprint: sqlalchemy-metadata-query
Related-Bug: #1093625
Change-Id: I2076e67b79448f98124a57b62b5bfed7aa8ae2ad
(cherry picked from commit 1570462507eae1478123de25dbadc64b09c82af3)
---
ceilometer/storage/impl_sqlalchemy.py | 79 +++++++++++++++++++---
.../versions/020_add_metadata_tables.py | 78 +++++++++++++++++++++
ceilometer/storage/sqlalchemy/models.py | 48 +++++++++++++
ceilometer/utils.py | 24 +++++++
doc/source/install/dbreco.rst | 4 +-
tests/api/v2/test_list_meters_scenarios.py | 1 +
tests/test_utils.py | 16 +++++
7 files changed, 239 insertions(+), 11 deletions(-)
create mode 100644 ceilometer/storage/sqlalchemy/migrate_repo/versions/020_add_metadata_tables.py
diff --git a/ceilometer/storage/impl_sqlalchemy.py b/ceilometer/storage/impl_sqlalchemy.py
index 08ffb47..9ede0f3 100644
--- a/ceilometer/storage/impl_sqlalchemy.py
+++ b/ceilometer/storage/impl_sqlalchemy.py
@@ -18,10 +18,12 @@
"""SQLAlchemy storage backend."""
from __future__ import absolute_import
-
import datetime
import operator
import os
+import types
+
+from sqlalchemy import and_
from sqlalchemy import func
from sqlalchemy import desc
from sqlalchemy.orm import aliased
@@ -39,6 +41,10 @@ from ceilometer.storage.sqlalchemy.models import AlarmChange
from ceilometer.storage.sqlalchemy.models import Base
from ceilometer.storage.sqlalchemy.models import Event
from ceilometer.storage.sqlalchemy.models import Meter
+from ceilometer.storage.sqlalchemy.models import MetaBool
+from ceilometer.storage.sqlalchemy.models import MetaFloat
+from ceilometer.storage.sqlalchemy.models import MetaInt
+from ceilometer.storage.sqlalchemy.models import MetaText
from ceilometer.storage.sqlalchemy.models import Project
from ceilometer.storage.sqlalchemy.models import Resource
from ceilometer.storage.sqlalchemy.models import Source
@@ -100,7 +106,40 @@ class SQLAlchemyStorage(base.StorageEngine):
return Connection(conf)
-def make_query_from_filter(query, sample_filter, require_meter=True):
+META_TYPE_MAP = {bool: MetaBool,
+ str: MetaText,
+ unicode: MetaText,
+ types.NoneType: MetaText,
+ int: MetaInt,
+ long: MetaInt,
+ float: MetaFloat}
+
+
+def apply_metaquery_filter(session, query, metaquery):
+ """Apply provided metaquery filter to existing query.
+
+ :param session: session used for original query
+ :param query: Query instance
+ :param metaquery: dict with metadata to match on.
+ """
+
+ for k, v in metaquery.iteritems():
+ key = k[9:] # strip out 'metadata.' prefix
+ try:
+ _model = META_TYPE_MAP[type(v)]
+ except KeyError:
+ raise NotImplementedError(_('Query on %(key)s is of %(value)s '
+ 'type and is not supported') %
+ {"key": k, "value": type(v)})
+ else:
+ meta_q = session.query(_model).\
+ filter(and_(_model.meta_key == key,
+ _model.value == v)).subquery()
+ query = query.filter_by(id=meta_q.c.id)
+ return query
+
+
+def make_query_from_filter(session, query, sample_filter, require_meter=True):
"""Return a query dictionary based on the settings in the filter.
:param filter: SampleFilter instance
@@ -134,7 +173,8 @@ def make_query_from_filter(query, sample_filter, require_meter=True):
query = query.filter_by(resource_id=sample_filter.resource)
if sample_filter.metaquery:
- raise NotImplementedError(_('metaquery not implemented'))
+ query = apply_metaquery_filter(session, query,
+ sample_filter.metaquery)
return query
@@ -229,6 +269,21 @@ class Connection(base.Connection):
meter.message_signature = data['message_signature']
meter.message_id = data['message_id']
+ if rmetadata:
+ if isinstance(rmetadata, dict):
+ for key, v in utils.dict_to_keyval(rmetadata):
+ try:
+ _model = META_TYPE_MAP[type(v)]
+ except KeyError:
+ LOG.warn(_("Unknown metadata type. Key (%s) will "
+ "not be queryable."), key)
+ else:
+ session.add(_model(id=meter.id,
+ meta_key=key,
+ value=v))
+
+ session.flush()
+
@staticmethod
def clear_expired_metering_data(ttl):
"""Clear expired data from the backend storage system according to the
@@ -306,8 +361,6 @@ class Connection(base.Connection):
# just fail.
if pagination:
raise NotImplementedError(_('Pagination not implemented'))
- if metaquery:
- raise NotImplementedError(_('metaquery not implemented'))
# (thomasm) We need to get the max timestamp first, since that's the
# most accurate. We also need to filter down in the subquery to
@@ -331,6 +384,11 @@ class Connection(base.Connection):
ts_subquery = ts_subquery.filter(
Meter.sources.any(id=source))
+ if metaquery:
+ ts_subquery = apply_metaquery_filter(session,
+ ts_subquery,
+ metaquery)
+
# Here we limit the samples being used to a specific time period,
# if requested.
if start_timestamp:
@@ -397,8 +455,6 @@ class Connection(base.Connection):
if pagination:
raise NotImplementedError(_('Pagination not implemented'))
- if metaquery:
- raise NotImplementedError(_('metaquery not implemented'))
session = sqlalchemy_session.get_session()
@@ -422,6 +478,11 @@ class Connection(base.Connection):
query_meter = session.query(Meter).\
join(subquery_meter, Meter.id == subquery_meter.c.id)
+ if metaquery:
+ query_meter = apply_metaquery_filter(session,
+ query_meter,
+ metaquery)
+
alias_meter = aliased(Meter, query_meter.subquery())
query = session.query(Resource, alias_meter).join(
alias_meter, Resource.id == alias_meter.resource_id)
@@ -457,7 +518,7 @@ class Connection(base.Connection):
session = sqlalchemy_session.get_session()
query = session.query(Meter)
- query = make_query_from_filter(query, sample_filter,
+ query = make_query_from_filter(session, query, sample_filter,
require_meter=False)
if limit:
query = query.limit(limit)
@@ -509,7 +570,7 @@ class Connection(base.Connection):
if groupby:
query = query.group_by(*group_attributes)
- return make_query_from_filter(query, sample_filter)
+ return make_query_from_filter(session, query, sample_filter)
@staticmethod
def _stats_result_to_model(result, period, period_start,
diff --git a/ceilometer/storage/sqlalchemy/migrate_repo/versions/020_add_metadata_tables.py b/ceilometer/storage/sqlalchemy/migrate_repo/versions/020_add_metadata_tables.py
new file mode 100644
index 0000000..085cd6b
--- /dev/null
+++ b/ceilometer/storage/sqlalchemy/migrate_repo/versions/020_add_metadata_tables.py
@@ -0,0 +1,78 @@
+#
+# Copyright 2013 OpenStack Foundation
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+import json
+
+from sqlalchemy import Boolean
+from sqlalchemy import Column
+from sqlalchemy import Float
+from sqlalchemy import ForeignKey
+from sqlalchemy import Integer
+from sqlalchemy import MetaData
+from sqlalchemy import String
+from sqlalchemy import Table
+from sqlalchemy import Text
+from sqlalchemy.sql import select
+
+from ceilometer import utils
+
+tables = [('metadata_text', Text, True),
+ ('metadata_bool', Boolean, False),
+ ('metadata_int', Integer, False),
+ ('metadata_float', Float, False)]
+
+
+def upgrade(migrate_engine):
+ meta = MetaData(bind=migrate_engine)
+ meter = Table('meter', meta, autoload=True)
+ meta_tables = {}
+ for t_name, t_type, t_nullable in tables:
+ meta_tables[t_name] = Table(
+ t_name, meta,
+ Column('id', Integer, ForeignKey('meter.id'), primary_key=True),
+ Column('meta_key', String(255), index=True, primary_key=True),
+ Column('value', t_type, nullable=t_nullable),
+ mysql_engine='InnoDB',
+ mysql_charset='utf8',
+ )
+ meta_tables[t_name].create()
+
+ for row in select([meter]).execute():
+ meter_id = row['id']
+ rmeta = json.loads(row['resource_metadata'])
+ for key, v in utils.dict_to_keyval(rmeta):
+ if isinstance(v, basestring) or v is None:
+ meta_tables['metadata_text'].insert().values(id=meter_id,
+ meta_key=key,
+ value=v)
+ elif isinstance(v, bool):
+ meta_tables['metadata_bool'].insert().values(id=meter_id,
+ meta_key=key,
+ value=v)
+ elif isinstance(v, (int, long)):
+ meta_tables['metadata_int'].insert().values(id=meter_id,
+ meta_key=key,
+ value=v)
+ elif isinstance(v, float):
+ meta_tables['metadata_float'].insert().values(id=meter_id,
+ meta_key=key,
+ value=v)
+
+
+def downgrade(migrate_engine):
+ meta = MetaData(bind=migrate_engine)
+ for t in tables:
+ table = Table(t[0], meta, autoload=True)
+ table.drop()
diff --git a/ceilometer/storage/sqlalchemy/models.py b/ceilometer/storage/sqlalchemy/models.py
index 45f98cb..8f890b3 100644
--- a/ceilometer/storage/sqlalchemy/models.py
+++ b/ceilometer/storage/sqlalchemy/models.py
@@ -141,6 +141,54 @@ class Source(Base):
id = Column(String(255), primary_key=True)
+class MetaText(Base):
+ """Metering text metadata."""
+
+ __tablename__ = 'metadata_text'
+ __table_args__ = (
+ Index('ix_meta_text_key', 'meta_key'),
+ )
+ id = Column(Integer, ForeignKey('meter.id'), primary_key=True)
+ meta_key = Column(String(255), primary_key=True)
+ value = Column(Text)
+
+
+class MetaBool(Base):
+ """Metering boolean metadata."""
+
+ __tablename__ = 'metadata_bool'
+ __table_args__ = (
+ Index('ix_meta_bool_key', 'meta_key'),
+ )
+ id = Column(Integer, ForeignKey('meter.id'), primary_key=True)
+ meta_key = Column(String(255), primary_key=True)
+ value = Column(Boolean)
+
+
+class MetaInt(Base):
+ """Metering integer metadata."""
+
+ __tablename__ = 'metadata_int'
+ __table_args__ = (
+ Index('ix_meta_int_key', 'meta_key'),
+ )
+ id = Column(Integer, ForeignKey('meter.id'), primary_key=True)
+ meta_key = Column(String(255), primary_key=True)
+ value = Column(Integer, default=False)
+
+
+class MetaFloat(Base):
+ """Metering float metadata."""
+
+ __tablename__ = 'metadata_float'
+ __table_args__ = (
+ Index('ix_meta_float_key', 'meta_key'),
+ )
+ id = Column(Integer, ForeignKey('meter.id'), primary_key=True)
+ meta_key = Column(String(255), primary_key=True)
+ value = Column(Float, default=False)
+
+
class Meter(Base):
"""Metering data."""
diff --git a/ceilometer/utils.py b/ceilometer/utils.py
index 07faf29..0bc7dfc 100644
--- a/ceilometer/utils.py
+++ b/ceilometer/utils.py
@@ -90,3 +90,27 @@ def stringify_timestamps(data):
isa_timestamp = lambda v: isinstance(v, datetime.datetime)
return dict((k, v.isoformat() if isa_timestamp(v) else v)
for (k, v) in data.iteritems())
+
+
+def dict_to_keyval(value, key_base=None):
+ """Expand a given dict to its corresponding key-value pairs.
+
+ Generated keys are fully qualified, delimited using dot notation.
+ ie. key = 'key.child_key.grandchild_key[0]'
+ """
+ val_iter, key_func = None, None
+ if isinstance(value, dict):
+ val_iter = value.iteritems()
+ key_func = lambda k: key_base + '.' + k if key_base else k
+ elif isinstance(value, (tuple, list)):
+ val_iter = enumerate(value)
+ key_func = lambda k: key_base + '[%d]' % k
+
+ if val_iter:
+ for k, v in val_iter:
+ key_gen = key_func(k)
+ if isinstance(v, dict) or isinstance(v, (tuple, list)):
+ for key_gen, v in dict_to_keyval(v, key_gen):
+ yield key_gen, v
+ else:
+ yield key_gen, v
diff --git a/doc/source/install/dbreco.rst b/doc/source/install/dbreco.rst
index fe60329..249cdc7 100644
--- a/doc/source/install/dbreco.rst
+++ b/doc/source/install/dbreco.rst
@@ -43,8 +43,8 @@ The following is a table indicating the status of each database drivers:
Driver API querying API statistics Alarms
================== ============================= =================== ======
MongoDB Yes Yes Yes
-MySQL Yes, except metadata querying Yes Yes
-PostgreSQL Yes, except metadata querying Yes Yes
+MySQL Yes Yes Yes
+PostgreSQL Yes Yes Yes
HBase Yes Yes, except groupby No
DB2 Yes Yes No
================== ============================= =================== ======
diff --git a/tests/api/v2/test_list_meters_scenarios.py b/tests/api/v2/test_list_meters_scenarios.py
index fe2c5b7..3381e15 100644
--- a/tests/api/v2/test_list_meters_scenarios.py
+++ b/tests/api/v2/test_list_meters_scenarios.py
@@ -252,6 +252,7 @@ class TestListMeters(FunctionalTest,
set(['meter.mine']))
self.assertEqual(set(r['resource_metadata']['is_public'] for r
in data), set(['False']))
+ # FIXME(gordc): verify no false positive (Bug#1236496)
def test_list_meters_query_string_metadata(self):
data = self.get_json('/meters/meter.test',
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 90e06d6..ec60542 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -84,3 +84,19 @@ class TestUtils(tests_base.TestCase):
def test_decimal_to_dt_with_none_parameter(self):
self.assertEqual(utils.decimal_to_dt(None), None)
+
+ def test_dict_to_kv(self):
+ data = {'a': 'A',
+ 'b': 'B',
+ 'nested': {'a': 'A',
+ 'b': 'B',
+ },
+ 'nested2': [{'c': 'A'}, {'c': 'B'}]
+ }
+ pairs = list(utils.dict_to_keyval(data))
+ self.assertEqual(pairs, [('a', 'A'),
+ ('b', 'B'),
+ ('nested2[0].c', 'A'),
+ ('nested2[1].c', 'B'),
+ ('nested.a', 'A'),
+ ('nested.b', 'B')])
--
1.8.4.1
