Skip to content
Commits on Source (6)
include gittaggers.py Makefile LICENSE.txt requirements.txt
include schema_salad/py.typed
include schema_salad/avro/*
recursive-include schema_salad/java *
include schema_salad/tests/*
include schema_salad/tests/test_schema/*.md
include schema_salad/tests/test_schema/*.yml
......
......@@ -33,7 +33,7 @@ COVBASE=coverage run --branch --append --source=${MODULE} \
# Updating the Major & Minor version below?
# Don't forget to update setup.py as well
VERSION=4.5.$(shell date +%Y%m%d%H%M%S --utc --date=`git log --first-parent \
VERSION=5.0.$(shell date +%Y%m%d%H%M%S --utc --date=`git log --first-parent \
--max-count=1 --format=format:%cI`)
## all : default task
......@@ -86,7 +86,7 @@ diff_pydocstyle_report: pydocstyle_report.txt
## format : check/fix all code indentation and formatting (runs black)
format:
black --target-version py27 schema_salad
black --exclude metaschema.py schema_salad
## pylint : run static code analysis on Python code
pylint: $(PYSOURCES)
......@@ -161,22 +161,12 @@ list-author-emails:
@echo 'name, E-Mail Address'
@git log --format='%aN,%aE' | sort -u | grep -v 'root'
mypy2: ${PYSOURCES}
if ! test -f $(shell python -c 'from __future__ import print_function; import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))')/py.typed ; \
mypy3: mypy
mypy: ${PYSOURCES}
if ! test -f $(shell python3 -c 'import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))')/py.typed ; \
then \
rm -Rf typeshed/2and3/ruamel/yaml ; \
ln -s $(shell python -c 'from __future__ import print_function; import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \
typeshed/2and3/ruamel/ ; \
fi # if minimally required ruamel.yaml version is 0.15.99 or greater, than the above can be removed
MYPYPATH=$$MYPYPATH:typeshed/2.7:typeshed/2and3 mypy --py2 --disallow-untyped-calls \
--warn-redundant-casts \
schema_salad
mypy3: ${PYSOURCES}
if ! test -f $(shell python -c 'from __future__ import print_function; import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))')/py.typed ; \
then \
rm -Rf typeshed/2and3/ruamel/yaml ; \
ln -s $(shell python -c 'from __future__ import print_function; import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \
ln -s $(shell python3 -c 'import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \
typeshed/2and3/ruamel/ ; \
fi # if minimally required ruamel.yaml version is 0.15.99 or greater, than the above can be removed
MYPYPATH=$$MYPYPATH:typeshed/3:typeshed/2and3 mypy --disallow-untyped-calls \
......@@ -193,20 +183,18 @@ jenkins: FORCE
. env3/bin/activate ; \
pip install -U setuptools pip wheel ; \
${MAKE} install-dep ; \
pip install -U -r mypy_requirements.txt ; ${MAKE} mypy2
# pip install -U -r mypy_requirements.txt ; ${MAKE} mypy3
pip install -U -r mypy_requirements.txt ; ${MAKE} mypy
release-test: FORCE
git diff-index --quiet HEAD -- || ( echo You have uncommited changes, please commit them and try again; false )
PYVER=2.7 ./release-test.sh
PYVER=3 ./release-test.sh
release: release-test
. testenv2.7_2/bin/activate && \
testenv2.7_2/src/${PACKAGE}/setup.py sdist bdist_wheel
. testenv2.7_2/bin/activate && \
. testenv3_2/bin/activate && \
testenv3_2/src/${PACKAGE}/setup.py sdist bdist_wheel
. testenv3.7_2/bin/activate && \
pip install twine && \
twine upload testenv2.7_2/src/${PACKAGE}/dist/* && \
twine upload testenv3_2/src/${PACKAGE}/dist/* && \
git tag ${VERSION} && git push --tags
FORCE:
......
Metadata-Version: 1.1
Metadata-Version: 2.1
Name: schema-salad
Version: 4.5.20190815125611
Version: 5.0.20200122085940
Summary: Schema Annotations for Linked Avro Data (SALAD)
Home-page: https://github.com/common-workflow-language/schema_salad
Author: Common workflow language working group
......@@ -31,6 +31,8 @@ Description: |Linux Build Status| |Windows Build status| |Code coverage| |CII Be
between document and record oriented data modeling and the Semantic
Web.
The Schema Salad library is Python 3.5+ only.
Usage
-----
......@@ -42,7 +44,7 @@ Description: |Linux Build Status| |Windows Build status| |Code coverage| |CII Be
git clone https://github.com/common-workflow-language/schema_salad
cd schema_salad
python setup.py install
python3 setup.py install
Commands
--------
......@@ -230,9 +232,11 @@ Classifier: Operating System :: POSIX
Classifier: Operating System :: MacOS :: MacOS X
Classifier: Operating System :: Microsoft :: Windows
Classifier: Development Status :: 5 - Production/Stable
Classifier: Programming Language :: Python :: 2.7
Classifier: Programming Language :: Python :: 3.5
Classifier: Programming Language :: Python :: 3.6
Classifier: Programming Language :: Python :: 3.7
Classifier: Programming Language :: Python :: 3.8
Classifier: Typing :: Typed
Requires-Python: >=3.5
Description-Content-Type: text/x-rst
Provides-Extra: docs
......@@ -22,6 +22,8 @@ generation, and transformation to RDF_. Salad provides a bridge
between document and record oriented data modeling and the Semantic
Web.
The Schema Salad library is Python 3.5+ only.
Usage
-----
......@@ -33,7 +35,7 @@ To install from source::
git clone https://github.com/common-workflow-language/schema_salad
cd schema_salad
python setup.py install
python3 setup.py install
Commands
--------
......
python-schema-salad (5.0.20200122085940-1) unstable; urgency=medium
* New upstream version
* Use secure URI in Homepage field.
-- Michael R. Crusoe <michael.crusoe@gmail.com> Fri, 24 Jan 2020 16:52:41 +0100
python-schema-salad (4.5.20190815125611-3) unstable; urgency=medium
* debian/patches/lower-case.patch: improvement from Steve Langasek
......
......@@ -21,10 +21,10 @@ Build-Depends: debhelper-compat (= 12),
python3-pytest-xdist,
python3-pytest-runner,
help2man
Standards-Version: 4.4.0
Standards-Version: 4.4.5
Vcs-Browser: https://salsa.debian.org/med-team/python-schema-salad
Vcs-Git: https://salsa.debian.org/med-team/python-schema-salad.git
Homepage: http://www.commonwl.org
Homepage: https://www.commonwl.org
Package: python3-schema-salad
Architecture: all
......@@ -32,7 +32,6 @@ Depends: ${python3:Depends},
${misc:Depends},
libjs-bootstrap
Recommends: cwltool
Breaks: cwltool (<< 1.0.20190621)
Conflicts: python-schema-salad
Description: Schema Annotations for Linked Avro Data (SALAD)
Salad is a schema language for describing JSON or YAML structured linked data
......
commit b88e74f8c4c041f1d4d15c56193b07dcddb95400
Author: Michael R. Crusoe <michael.crusoe@gmail.com>
Author: Steve Langasek <steve.langasek@canonical.com>
Date: Fri Aug 16 12:25:07 2019 +0200
only be case insensitive when needed
Index: python-schema-salad-4.5.20190815125611/schema_salad/tests/test_ref_resolver.py
===================================================================
--- python-schema-salad-4.5.20190815125611.orig/schema_salad/tests/test_ref_resolver.py
+++ python-schema-salad-4.5.20190815125611/schema_salad/tests/test_ref_resolver.py
@@ -13,6 +13,12 @@
from schema_salad.ref_resolver import DefaultFetcher, Loader, file_uri
from schema_salad.tests.util import get_data
+def is_fs_case_sensitive(path): # https://stackoverflow.com/a/36612604/1585509
+ try:
+ with tempfile.NamedTemporaryFile(prefix='TmP',dir=path) as tmp_file:
+ return(not os.path.exists(tmp_file.name.lower()))
+ except PermissionError:
+ return True
@pytest.fixture
def tmp_dir_fixture(request):
@@ -170,24 +176,27 @@
def test_fetch_inject_id():
+ lower = lambda s : s.lower()
+ if is_fs_case_sensitive(os.path.dirname(get_data("schema_salad/tests/inject-id1.yml"))):
+ lower = lambda a : a
l1 = Loader({"id": "@id"})
- furi1 = file_uri(get_data("schema_salad/tests/inject-id1.yml")).lower()
+ furi1 = file_uri(get_data("schema_salad/tests/inject-id1.yml"))
r1, _ = l1.resolve_ref(furi1)
assert {"id": furi1 + "#foo", "bar": "baz"} == r1
- assert [furi1, furi1 + "#foo"] == sorted(list(k.lower() for k in l1.idx.keys()))
+ assert [lower(furi1), furi1 + "#foo"] == sorted(list(lower(k) for k in l1.idx.keys()))
l2 = Loader({"id": "@id"})
- furi2 = file_uri(get_data("schema_salad/tests/inject-id2.yml")).lower()
+ furi2 = file_uri(get_data("schema_salad/tests/inject-id2.yml"))
r2, _ = l2.resolve_ref(furi2)
assert {"id": furi2, "bar": "baz"} == r2
- assert [furi2] == sorted(list(k.lower() for k in l2.idx.keys()))
+ assert [lower(furi2)] == sorted(list(lower(k) for k in l2.idx.keys()))
l3 = Loader({"id": "@id"})
- furi3 = file_uri(get_data("schema_salad/tests/inject-id3.yml")).lower()
+ furi3 = file_uri(get_data("schema_salad/tests/inject-id3.yml"))
r3, _ = l3.resolve_ref(furi3)
assert {"id": "http://example.com", "bar": "baz"} == r3
- assert [furi3, "http://example.com"] == sorted(
- list(k.lower() for k in l3.idx.keys())
+ assert [lower(furi3), "http://example.com"] == sorted(
+ list(lower(k) for k in l3.idx.keys())
)
import subprocess
import time
import pkg_resources
from setuptools.command.egg_info import egg_info
......
typing==3.7.4 ; python_version < "3.5"
ruamel.yaml>=0.12.4, <= 0.16
ruamel.yaml>=0.12.4, <= 0.16.5
rdflib==4.2.2
rdflib-jsonld==0.4.0
mistune>=0.8.1,<0.9
CacheControl==0.11.7
lockfile==0.12.2
typing-extensions
future
Metadata-Version: 1.1
Metadata-Version: 2.1
Name: schema-salad
Version: 4.5.20190815125611
Version: 5.0.20200122085940
Summary: Schema Annotations for Linked Avro Data (SALAD)
Home-page: https://github.com/common-workflow-language/schema_salad
Author: Common workflow language working group
......@@ -31,6 +31,8 @@ Description: |Linux Build Status| |Windows Build status| |Code coverage| |CII Be
between document and record oriented data modeling and the Semantic
Web.
The Schema Salad library is Python 3.5+ only.
Usage
-----
......@@ -42,7 +44,7 @@ Description: |Linux Build Status| |Windows Build status| |Code coverage| |CII Be
git clone https://github.com/common-workflow-language/schema_salad
cd schema_salad
python setup.py install
python3 setup.py install
Commands
--------
......@@ -230,9 +232,11 @@ Classifier: Operating System :: POSIX
Classifier: Operating System :: MacOS :: MacOS X
Classifier: Operating System :: Microsoft :: Windows
Classifier: Development Status :: 5 - Production/Stable
Classifier: Programming Language :: Python :: 2.7
Classifier: Programming Language :: Python :: 3.5
Classifier: Programming Language :: Python :: 3.6
Classifier: Programming Language :: Python :: 3.7
Classifier: Programming Language :: Python :: 3.8
Classifier: Typing :: Typed
Requires-Python: >=3.5
Description-Content-Type: text/x-rst
Provides-Extra: docs
LICENSE.txt
MANIFEST.in
Makefile
PKG-INFO
README.rst
gittaggers.py
requirements.txt
setup.cfg
setup.py
debian/changelog
debian/clean
debian/control
debian/copyright
debian/lintian-overrides
debian/manpages
debian/rules
debian/watch
debian/source/format
debian/source/options
debian/tests/control
debian/tests/run-tests
schema_salad/__init__.py
schema_salad/__main__.py
schema_salad/codegen.py
schema_salad/codegen_base.py
schema_salad/exceptions.py
schema_salad/java_codegen.py
schema_salad/jsonld_context.py
schema_salad/main.py
......@@ -34,6 +48,43 @@ schema_salad/avro/LICENSE
schema_salad/avro/NOTICE
schema_salad/avro/__init__.py
schema_salad/avro/schema.py
schema_salad/java/MANIFEST.MF
schema_salad/java/README.md
schema_salad/java/gitignore
schema_salad/java/overview.html
schema_salad/java/package.html
schema_salad/java/pom.xml
schema_salad/java/main_utils/AnyLoader.java
schema_salad/java/main_utils/ArrayLoader.java
schema_salad/java/main_utils/ConstantMaps.java
schema_salad/java/main_utils/DefaultFetcher.java
schema_salad/java/main_utils/EnumLoader.java
schema_salad/java/main_utils/Fetcher.java
schema_salad/java/main_utils/IdMapLoader.java
schema_salad/java/main_utils/Loader.java
schema_salad/java/main_utils/LoaderInstances.java
schema_salad/java/main_utils/LoadingOptions.java
schema_salad/java/main_utils/LoadingOptionsBuilder.java
schema_salad/java/main_utils/NullLoader.java
schema_salad/java/main_utils/OneOrListOf.java
schema_salad/java/main_utils/OneOrListOfLoader.java
schema_salad/java/main_utils/OptionalLoader.java
schema_salad/java/main_utils/PrimitiveLoader.java
schema_salad/java/main_utils/RecordLoader.java
schema_salad/java/main_utils/RootLoader.java
schema_salad/java/main_utils/Savable.java
schema_salad/java/main_utils/SavableImpl.java
schema_salad/java/main_utils/TypeDslLoader.java
schema_salad/java/main_utils/UnionLoader.java
schema_salad/java/main_utils/UriLoader.java
schema_salad/java/main_utils/Uris.java
schema_salad/java/main_utils/ValidationException.java
schema_salad/java/main_utils/Validator.java
schema_salad/java/main_utils/YamlUtils.java
schema_salad/java/main_utils/package.html
schema_salad/java/test_utils/DefaultFetcherTest.java
schema_salad/java/test_utils/ExamplesTest.java
schema_salad/java/test_utils/YamlUtilsTest.java
schema_salad/metaschema/field_name.yml
schema_salad/metaschema/field_name_proc.yml
schema_salad/metaschema/field_name_schema.yml
......@@ -90,7 +141,9 @@ schema_salad/tests/test_errors.py
schema_salad/tests/test_examples.py
schema_salad/tests/test_fetch.py
schema_salad/tests/test_fp.py
schema_salad/tests/test_java_codegen.py
schema_salad/tests/test_print_oneline.py
schema_salad/tests/test_python_codegen.py
schema_salad/tests/test_real_cwl.py
schema_salad/tests/test_ref_resolver.py
schema_salad/tests/util.py
......
setuptools
requests>=1.0
ruamel.yaml<=0.16,>=0.12.4
rdflib<4.3.0,>=4.2.2
rdflib-jsonld<0.5.0,>=0.3.0
mistune<0.9,>=0.8.1
CacheControl<0.12,>=0.11.7
lockfile>=0.9
six>=1.8.0
mistune<0.9,>=0.8.1
rdflib-jsonld<0.5.0,>=0.3.0
rdflib<4.3.0,>=4.2.2
requests>=1.0
ruamel.yaml<=0.16.5,>=0.12.4
setuptools
typing-extensions
future
[:python_version<"3.5"]
typing>=3.7.4
[docs]
pytest
sphinx-rtd-theme
sphinx>=2.2
from __future__ import absolute_import
"""Salad is a schema language for describing JSON or YAML structured linked data documents"""
import logging
......
"""Default entry point for the schema-salad module."""
from __future__ import absolute_import
import sys
......
......@@ -30,9 +30,9 @@ A schema may be one of:
A boolean; or
Null.
"""
from typing import Any, Dict, List, Optional, Text, Tuple, Union, cast
from typing import Any, Dict, List, Optional, Tuple, Union, cast
import six
from schema_salad.exceptions import SchemaException
#
# Constants
......@@ -63,7 +63,7 @@ VALID_FIELD_SORT_ORDERS = ("ascending", "descending", "ignore")
#
class AvroException(Exception):
class AvroException(SchemaException):
pass
......@@ -80,9 +80,9 @@ class Schema(object):
"""Base class for all Schema classes."""
def __init__(self, atype, other_props=None):
# type: (Text, Optional[Dict[Text, Any]]) -> None
# type: (str, Optional[Dict[str, Any]]) -> None
# Ensure valid ctor args
if not isinstance(atype, six.string_types):
if not isinstance(atype, str):
raise SchemaParseException(
"Schema type '{}' must be a string, was '{}.".format(atype, type(atype))
)
......@@ -92,7 +92,7 @@ class Schema(object):
# add members
if not hasattr(self, "_props"):
self._props = {} # type: Dict[Text, Any]
self._props = {} # type: Dict[str, Any]
self.set_prop("type", atype)
self.type = atype
self._props.update(other_props or {})
......@@ -102,10 +102,10 @@ class Schema(object):
props = property(lambda self: self._props)
# utility functions to manipulate properties dict
def get_prop(self, key): # type: (Text) -> Any
def get_prop(self, key): # type: (str) -> Any
return self._props.get(key)
def set_prop(self, key, value): # type: (Text, Any) -> None
def set_prop(self, key, value): # type: (str, Any) -> None
self._props[key] = value
......@@ -113,7 +113,7 @@ class Name(object):
"""Class to describe Avro name."""
def __init__(self, name_attr, space_attr, default_space):
# type: (Text, Optional[Text], Optional[Text]) -> None
# type: (str, Optional[str], Optional[str]) -> None
"""
Formulate full name according to the specification.
......@@ -122,28 +122,28 @@ class Name(object):
@ard default_space: the current default space or None.
"""
# Ensure valid ctor args
if not (isinstance(name_attr, six.string_types) or (name_attr is None)):
if not (isinstance(name_attr, str) or (name_attr is None)):
fail_msg = "Name must be non-empty string or None."
raise SchemaParseException(fail_msg)
elif name_attr == "":
fail_msg = "Name must be non-empty string or None."
raise SchemaParseException(fail_msg)
if not (isinstance(space_attr, six.string_types) or (space_attr is None)):
if not (isinstance(space_attr, str) or (space_attr is None)):
fail_msg = "Space must be non-empty string or None."
raise SchemaParseException(fail_msg)
elif name_attr == "":
fail_msg = "Space must be non-empty string or None."
raise SchemaParseException(fail_msg)
if not (isinstance(default_space, six.string_types) or (default_space is None)):
if not (isinstance(default_space, str) or (default_space is None)):
fail_msg = "Default space must be non-empty string or None."
raise SchemaParseException(fail_msg)
elif name_attr == "":
fail_msg = "Default must be non-empty string or None."
raise SchemaParseException(fail_msg)
self._full = None # type: Optional[Text]
self._full = None # type: Optional[str]
if name_attr is None or name_attr == "":
return
......@@ -162,7 +162,7 @@ class Name(object):
fullname = property(lambda self: self._full)
def get_space(self):
# type: () -> Optional[Text]
# type: () -> Optional[str]
"""Back out a namespace from full name."""
if self._full is None:
return None
......@@ -177,24 +177,24 @@ class Names(object):
"""Track name set and default namespace during parsing."""
def __init__(self, default_namespace=None):
# type: (Optional[Text]) -> None
self.names = {} # type: Dict[Text, NamedSchema]
# type: (Optional[str]) -> None
self.names = {} # type: Dict[str, NamedSchema]
self.default_namespace = default_namespace
def has_name(self, name_attr, space_attr):
# type: (Text, Optional[Text]) -> bool
# type: (str, Optional[str]) -> bool
test = Name(name_attr, space_attr, self.default_namespace).fullname
return test in self.names
def get_name(self, name_attr, space_attr):
# type: (Text, Optional[Text]) -> Optional[NamedSchema]
# type: (str, Optional[str]) -> Optional[NamedSchema]
test = Name(name_attr, space_attr, self.default_namespace).fullname
if test not in self.names:
return None
return self.names[test]
def add_name(self, name_attr, space_attr, new_schema):
# type: (Text, Optional[Text], NamedSchema) -> Name
# type: (str, Optional[str], NamedSchema) -> Name
"""
Add a new schema object to the name set.
......@@ -221,20 +221,20 @@ class NamedSchema(Schema):
def __init__(
self,
atype, # type: Text
name, # type: Text
namespace=None, # type: Optional[Text]
atype, # type: str
name, # type: str
namespace=None, # type: Optional[str]
names=None, # type: Optional[Names]
other_props=None, # type: Optional[Dict[Text, Text]]
other_props=None, # type: Optional[Dict[str, str]]
): # type: (...) -> None
# Ensure valid ctor args
if not name:
fail_msg = "Named Schemas must have a non-empty name."
raise SchemaParseException(fail_msg)
elif not isinstance(name, six.string_types):
elif not isinstance(name, str):
fail_msg = "The name property must be a string."
raise SchemaParseException(fail_msg)
elif namespace is not None and not isinstance(namespace, six.string_types):
elif namespace is not None and not isinstance(namespace, str):
fail_msg = "The namespace property must be a string."
raise SchemaParseException(fail_msg)
if names is None:
......@@ -261,20 +261,20 @@ class NamedSchema(Schema):
class Field(object):
def __init__(
self,
atype, # type: Union[Text, Dict[Text, Text]]
name, # type: Text
atype, # type: Union[str, Dict[str, str]]
name, # type: str
has_default, # type: bool
default=None, # type: Optional[Text]
order=None, # type: Optional[Text]
default=None, # type: Optional[str]
order=None, # type: Optional[str]
names=None, # type: Optional[Names]
doc=None, # type: Optional[Text]
other_props=None, # type: Optional[Dict[Text, Text]]
doc=None, # type: Optional[str]
other_props=None, # type: Optional[Dict[str, str]]
): # type: (...) -> None
# Ensure valid ctor args
if not name:
fail_msg = "Fields must have a non-empty name."
raise SchemaParseException(fail_msg)
elif not isinstance(name, six.string_types):
elif not isinstance(name, str):
fail_msg = "The name property must be a string."
raise SchemaParseException(fail_msg)
elif order is not None and order not in VALID_FIELD_SORT_ORDERS:
......@@ -282,19 +282,15 @@ class Field(object):
raise SchemaParseException(fail_msg)
# add members
self._props = {} # type: Dict[Text, Union[Schema, Text, None]]
self._props = {} # type: Dict[str, Union[Schema, str, None]]
self._has_default = has_default
self._props.update(other_props or {})
if (
isinstance(atype, six.string_types)
and names is not None
and names.has_name(atype, None)
):
if isinstance(atype, str) and names is not None and names.has_name(atype, None):
type_schema = cast(NamedSchema, names.get_name(atype, None)) # type: Schema
else:
try:
type_schema = make_avsc_object(cast(Dict[Text, Text], atype), names)
type_schema = make_avsc_object(cast(Dict[str, str], atype), names)
except Exception as e:
raise SchemaParseException(
'Type property "%s" not a valid Avro schema: %s' % (atype, e)
......@@ -315,11 +311,11 @@ class Field(object):
default = property(lambda self: self.get_prop("default"))
# utility functions to manipulate properties dict
def get_prop(self, key): # type: (Text) -> Union[Schema, Text, None]
def get_prop(self, key): # type: (str) -> Union[Schema, str, None]
return self._props.get(key)
def set_prop(self, key, value):
# type: (Text, Union[Schema, Text, None]) -> None
# type: (str, Union[Schema, str, None]) -> None
self._props[key] = value
......@@ -330,7 +326,7 @@ class PrimitiveSchema(Schema):
"""Valid primitive types are in PRIMITIVE_TYPES."""
def __init__(self, atype, other_props=None):
# type: (Text, Optional[Dict[Text, Text]]) -> None
# type: (str, Optional[Dict[str, str]]) -> None
# Ensure valid ctor args
if atype not in PRIMITIVE_TYPES:
raise AvroException("%s is not a valid primitive type." % atype)
......@@ -349,18 +345,18 @@ class PrimitiveSchema(Schema):
class EnumSchema(NamedSchema):
def __init__(
self,
name, # type: Text
namespace, # type: Text
symbols, # type: List[Text]
name, # type: str
namespace, # type: str
symbols, # type: List[str]
names=None, # type: Optional[Names]
doc=None, # type: Optional[Text]
other_props=None, # type: Optional[Dict[Text, Text]]
doc=None, # type: Optional[str]
other_props=None, # type: Optional[Dict[str, str]]
): # type: (...) -> None
# Ensure valid ctor args
if not isinstance(symbols, list):
fail_msg = "Enum Schema requires a JSON array for the symbols property."
raise AvroException(fail_msg)
elif False in [isinstance(s, six.string_types) for s in symbols]:
elif False in [isinstance(s, str) for s in symbols]:
fail_msg = "Enum Schema requires all symbols to be JSON strings."
raise AvroException(fail_msg)
elif len(set(symbols)) < len(symbols):
......@@ -386,14 +382,14 @@ class EnumSchema(NamedSchema):
class ArraySchema(Schema):
def __init__(self, items, names=None, other_props=None):
# type: (List[Any], Optional[Names], Optional[Dict[Text, Text]]) -> None
# type: (List[Any], Optional[Names], Optional[Dict[str, str]]) -> None
# Call parent ctor
Schema.__init__(self, "array", other_props)
# Add class members
if names is None:
raise SchemaParseException("Must provide Names.")
if isinstance(items, six.string_types) and names.has_name(items, None):
if isinstance(items, str) and names.has_name(items, None):
items_schema = cast(Schema, names.get_name(items, None))
else:
try:
......@@ -430,14 +426,14 @@ class UnionSchema(Schema):
# Add class members
schema_objects = [] # type: List[Schema]
for schema in schemas:
if isinstance(schema, six.string_types) and names.has_name(schema, None):
if isinstance(schema, str) and names.has_name(schema, None):
new_schema = cast(Schema, names.get_name(schema, None))
else:
try:
new_schema = make_avsc_object(schema, names) # type: ignore
except Exception as err:
raise SchemaParseException(
"Union item must be a valid Avro schema: %s" % Text(err)
"Union item must be a valid Avro schema: %s" % str(err)
)
# check the new schema
if (
......@@ -459,14 +455,14 @@ class UnionSchema(Schema):
class RecordSchema(NamedSchema):
@staticmethod
def make_field_objects(field_data, names):
# type: (List[Dict[Text, Text]], Names) -> List[Field]
# type: (List[Dict[str, str]], Names) -> List[Field]
"""We're going to need to make message parameters too."""
field_objects = []
field_names = [] # type: List[Text]
field_names = [] # type: List[str]
for field in field_data:
if hasattr(field, "get") and callable(field.get):
atype = cast(Text, field.get("type"))
name = cast(Text, field.get("name"))
atype = cast(str, field.get("type"))
name = cast(str, field.get("name"))
# null values can have a default value of None
has_default = False
......@@ -493,13 +489,13 @@ class RecordSchema(NamedSchema):
def __init__(
self,
name, # type: Text
namespace, # type: Text
fields, # type: List[Dict[Text, Text]]
name, # type: str
namespace, # type: str
fields, # type: List[Dict[str, str]]
names=None, # type: Optional[Names]
schema_type="record", # type: Text
doc=None, # type: Optional[Text]
other_props=None, # type: Optional[Dict[Text, Text]]
schema_type="record", # type: str
doc=None, # type: Optional[str]
other_props=None, # type: Optional[Dict[str, str]]
): # type: (...) -> None
# Ensure valid ctor args
if fields is None:
......@@ -537,7 +533,7 @@ class RecordSchema(NamedSchema):
# Module Methods
#
def get_other_props(all_props, reserved_props):
# type: (Dict[Text, Text], Tuple[Text, ...]) -> Optional[Dict[Text, Text]]
# type: (Dict[str, str], Tuple[str, ...]) -> Optional[Dict[str, str]]
"""
Retrieve the non-reserved properties from a dictionary of properties
@args reserved_props: The set of reserved properties to exclude
......@@ -550,7 +546,7 @@ def get_other_props(all_props, reserved_props):
def make_avsc_object(json_data, names=None):
# type: (Union[Dict[Text, Text], List[Any], Text], Optional[Names]) -> Schema
# type: (Union[Dict[str, str], List[Any], str], Optional[Names]) -> Schema
"""
Build Avro Schema from data parsed out of JSON string.
......@@ -563,19 +559,19 @@ def make_avsc_object(json_data, names=None):
# JSON object (non-union)
if hasattr(json_data, "get") and callable(json_data.get): # type: ignore
assert isinstance(json_data, Dict)
atype = cast(Text, json_data.get("type"))
atype = cast(str, json_data.get("type"))
other_props = get_other_props(json_data, SCHEMA_RESERVED_PROPS)
if atype in PRIMITIVE_TYPES:
return PrimitiveSchema(atype, other_props)
if atype in NAMED_TYPES:
name = cast(Text, json_data.get("name"))
namespace = cast(Text, json_data.get("namespace", names.default_namespace))
name = cast(str, json_data.get("name"))
namespace = cast(str, json_data.get("namespace", names.default_namespace))
if atype == "enum":
symbols = cast(List[Text], json_data.get("symbols"))
symbols = cast(List[str], json_data.get("symbols"))
doc = json_data.get("doc")
return EnumSchema(name, namespace, symbols, names, doc, other_props)
if atype in ["record", "error"]:
fields = cast(List[Dict[Text, Text]], json_data.get("fields"))
fields = cast(List[Dict[str, str]], json_data.get("fields"))
doc = json_data.get("doc")
return RecordSchema(
name, namespace, fields, names, atype, doc, other_props
......@@ -583,7 +579,7 @@ def make_avsc_object(json_data, names=None):
raise SchemaParseException("Unknown Named Type: %s" % atype)
if atype in VALID_TYPES:
if atype == "array":
items = cast(List[Text], json_data.get("items"))
items = cast(List[str], json_data.get("items"))
return ArraySchema(items, names, other_props)
if atype is None:
raise SchemaParseException('No "type" property: %s' % json_data)
......@@ -593,7 +589,7 @@ def make_avsc_object(json_data, names=None):
return UnionSchema(json_data, names)
# JSON string (primitive)
if json_data in PRIMITIVE_TYPES:
return PrimitiveSchema(cast(Text, json_data))
return PrimitiveSchema(cast(str, json_data))
# not for us!
fail_msg = "Could not make an Avro Schema object from %s." % json_data
raise SchemaParseException(fail_msg)
"""Generate langauge specific loaders for a particular SALAD schema."""
import sys
from io import open
from typing import Any, Dict, List, MutableMapping, Optional
from typing_extensions import Text # pylint: disable=unused-import
from . import schema
from .codegen_base import CodeGenBase
from .exceptions import SchemaSaladException
from .java_codegen import JavaCodeGen
from .python_codegen import PythonCodeGen
from .ref_resolver import Loader # pylint: disable=unused-import
from .ref_resolver import Loader
from .schema import shortname
from .utils import aslist
# move to a regular typing import when Python 3.3-3.6 is no longer supported
def codegen(
lang, # type: str
i, # type: List[Dict[Text, Any]]
schema_metadata, # type: Dict[Text, Any]
loader, # type: Loader
i, # type: List[Dict[str, Any]]
schema_metadata, # type: Dict[str, Any]
loader: Loader,
target=None, # type: Optional[str]
examples=None, # type: Optional[str]
): # type: (...) -> None
"""Generate classes with loaders for the given Schema Salad description."""
......@@ -27,11 +27,21 @@ def codegen(
gen = None # type: Optional[CodeGenBase]
if lang == "python":
gen = PythonCodeGen(sys.stdout)
if target:
dest = open(target, mode="w", encoding="utf-8")
else:
dest = sys.stdout
gen = PythonCodeGen(dest)
elif lang == "java":
gen = JavaCodeGen(schema_metadata.get("$base", schema_metadata.get("id")))
gen = JavaCodeGen(
schema_metadata.get("$base", schema_metadata.get("id")),
target=target,
examples=examples,
)
else:
raise Exception("Unsupported code generation language '{}'".format(lang))
raise SchemaSaladException(
"Unsupported code generation language '{}'".format(lang)
)
assert gen is not None
gen.prologue()
......
......@@ -2,33 +2,42 @@
import collections
from typing import Any, Dict, List, MutableSequence, Optional, Union
from typing_extensions import Text # pylint: disable=unused-import
from . import schema
# move to a regular typing import when Python 3.3-3.6 is no longer supported
class TypeDef(object): # pylint: disable=too-few-public-methods
"""Schema Salad type description."""
__slots__ = ["name", "init", "is_uri", "scoped_id", "ref_scope"]
__slots__ = [
"name",
"init",
"is_uri",
"scoped_id",
"ref_scope",
"loader_type",
"instance_type",
]
# switch to class-style typing.NamedTuple once support for Python < 3.6
# is dropped
def __init__(
self, # pylint: disable=too-many-arguments
name, # type: Text
init, # type: Text
name, # type: str
init, # type: str
is_uri=False, # type: bool
scoped_id=False, # type: bool
ref_scope=0, # type: Optional[int]
loader_type=None, # type: Optional[str]
instance_type=None, # type: Optional[str]
): # type: (...) -> None
self.name = name
self.init = init
self.is_uri = is_uri
self.scoped_id = scoped_id
self.ref_scope = ref_scope
# Follow attributes used by Java but not Python.
self.loader_type = loader_type
self.instance_type = instance_type
class CodeGenBase(object):
......@@ -37,8 +46,8 @@ class CodeGenBase(object):
def __init__(self): # type: () -> None
self.collected_types = (
collections.OrderedDict()
) # type: collections.OrderedDict[Text, TypeDef]
self.vocab = {} # type: Dict[Text, Text]
) # type: collections.OrderedDict[str, TypeDef]
self.vocab = {} # type: Dict[str, str]
def declare_type(self, declared_type): # type: (TypeDef) -> TypeDef
"""Add this type to our collection, if needed."""
......@@ -46,7 +55,7 @@ class CodeGenBase(object):
self.collected_types[declared_type.name] = declared_type
return declared_type
def add_vocab(self, name, uri): # type: (Text, Text) -> None
def add_vocab(self, name, uri): # type: (str, str) -> None
"""Add the given name as an abbreviation for the given URI."""
self.vocab[name] = uri
......@@ -55,39 +64,39 @@ class CodeGenBase(object):
raise NotImplementedError()
@staticmethod
def safe_name(name): # type: (Text) -> Text
def safe_name(name): # type: (str) -> str
"""Generate a safe version of the given name."""
return schema.avro_name(name)
def begin_class(
self, # pylint: disable=too-many-arguments
classname, # type: Text
extends, # type: MutableSequence[Text]
doc, # type: Text
classname, # type: str
extends, # type: MutableSequence[str]
doc, # type: str
abstract, # type: bool
field_names, # type: MutableSequence[Text]
idfield, # type: Text
field_names, # type: MutableSequence[str]
idfield, # type: str
): # type: (...) -> None
"""Produce the header for the given class."""
raise NotImplementedError()
def end_class(self, classname, field_names):
# type: (Text, List[Text]) -> None
# type: (str, List[str]) -> None
"""Signal that we are done with this class."""
raise NotImplementedError()
def type_loader(self, type_declaration):
# type: (Union[List[Any], Dict[Text, Any]]) -> TypeDef
# type: (Union[List[Any], Dict[str, Any]]) -> TypeDef
"""Parse the given type declaration and declare its components."""
raise NotImplementedError()
def declare_field(self, name, fieldtype, doc, optional):
# type: (Text, TypeDef, Text, bool) -> None
# type: (str, TypeDef, str, bool) -> None
"""Output the code to load the given field."""
raise NotImplementedError()
def declare_id_field(self, name, fieldtype, doc, optional):
# type: (Text, TypeDef, Text, bool) -> None
# type: (str, TypeDef, str, bool) -> None
"""Output the code to handle the given ID field."""
raise NotImplementedError()
......@@ -97,7 +106,7 @@ class CodeGenBase(object):
raise NotImplementedError()
def idmap_loader(self, field, inner, map_subject, map_predicate):
# type: (Text, TypeDef, Text, Union[Text, None]) -> TypeDef
# type: (str, TypeDef, str, Union[str, None]) -> TypeDef
"""Construct the TypeDef for the given mapped ID loader."""
raise NotImplementedError()
......
from typing import Any, List, Optional, Sequence, Tuple
from .sourceline import SourceLine, reflow_all, strip_duplicated_lineno
def to_one_line_messages(exc): # type: (SchemaSaladException) -> str
return "\n".join((c.summary() for c in exc.leaves()))
class SchemaSaladException(Exception):
"""Base class for all schema-salad exceptions."""
def __init__(
self,
msg, # type: str
sl=None, # type: Optional[SourceLine]
children=None, # type: Optional[Sequence[SchemaSaladException]]
bullet_for_children="", # type: str
): # type: (...) -> None
super(SchemaSaladException, self).__init__(msg)
self.message = self.args[0]
# It will be set by its parent
self.bullet = "" # type: str
def simplify(exc): # type: (SchemaSaladException) -> List[SchemaSaladException]
return [exc] if len(exc.message) else exc.children
def with_bullet(exc, bullet):
# type: (SchemaSaladException, str) -> SchemaSaladException
if exc.bullet == "":
exc.bullet = bullet
return exc
if children is None:
self.children = [] # type: List[SchemaSaladException]
elif len(children) <= 1:
self.children = sum((simplify(c) for c in children), [])
else:
self.children = sum(
(simplify(with_bullet(c, bullet_for_children)) for c in children), []
)
self.with_sourceline(sl)
self.propagate_sourceline()
def propagate_sourceline(self): # type: () -> None
if self.file is None:
return
for c in self.children:
if c.file is None:
c.file = self.file
c.start = self.start
c.end = self.end
c.propagate_sourceline()
def with_sourceline(
self, sl
): # type: (Optional[SourceLine]) -> SchemaSaladException
if sl and sl.file():
self.file = sl.file() # type: Optional[str]
self.start = sl.start() # type: Optional[Tuple[int, int]]
self.end = sl.end() # type: Optional[Tuple[int, int]]
else:
self.file = None
self.start = None
self.end = None
return self
def leaves(self): # type: () -> List[SchemaSaladException]
if len(self.children):
return sum((c.leaves() for c in self.children), [])
elif len(self.message):
return [self]
else:
return []
def prefix(self): # type: () -> str
if self.file:
linecol = self.start if self.start else ("", "") # type: Tuple[Any, Any]
return "{}:{}:{}: ".format(self.file, linecol[0], linecol[1])
else:
return ""
def summary(self, level=0, with_bullet=False): # type: (int, bool) -> str
indent_per_level = 2
spaces = (level * indent_per_level) * " "
bullet = self.bullet + " " if len(self.bullet) and with_bullet else ""
return "{}{}{}{}".format(self.prefix(), spaces, bullet, self.message)
def __str__(self): # type: () -> str
return str(self.pretty_str())
def pretty_str(self, level=0): # type: (int) -> str
if len(self.message):
my_summary = [self.summary(level, True)]
next_level = level + 1
else:
my_summary = []
next_level = level
ret = "\n".join(
(e for e in my_summary + [c.pretty_str(next_level) for c in self.children])
)
if level == 0:
return strip_duplicated_lineno(reflow_all(ret))
else:
return ret
class SchemaException(SchemaSaladException):
"""Indicates error with the provided schema definition."""
pass
class ValidationException(SchemaSaladException):
"""Indicates error with document against the provided schema."""
pass
class ClassValidationException(ValidationException):
pass
Manifest-Version: 1.0
Main-Class: ${package}.utils.Validator