sync with develop

Author: neil-tan

.circleci/config.yml

@@ -6,7 +6,7 @@ jobs:
     steps:
       - checkout
       - run: mkdir -p /tmp/coverage && mkdir -p /tmp/test_results
-      - run: pip3 install -e .[dev]
+      - run: pip3 install .
       - run: 
           name: Test
           command: pytest -vv tests

.gitignore

@@ -8,6 +8,8 @@ dev_notes
 !tests/deep_mlp/end_to_end.ipynb
 *.egg*
 worktree/
+build/
+dist/
 # tests
 .pytest_cache
 tests/*/cpp

MANIFEST.in

@@ -1,2 +1,2 @@
 include LICENSE README.md
-recursive-include utensor_cgen/backend/snippets/templates *
+recursive-include utensor_cgen/backend/utensor/snippets/templates *

Makefile

@@ -3,7 +3,7 @@
 tests:
 	rm -f tests_log.txt
 	make test_utils test_ir test_transformer test_frontend \
-		test_matcher test_graph_constructor
+		test_matcher test_graph_constructor test_backend
 
 test_%:
 	@if [ -d .venv ]; then \
@@ -12,5 +12,23 @@ test_%:
 	 	 pytest tests/$@ -vv -s | tee -a tests_log.txt; \
 	 fi;
 
+package:
+	rm -rf dist/*
+	.venv/bin/python setup.py bdist_wheel sdist
+	rm -rf build utensor_cgen.egg-info/
+
+upload-test: package
+	.venv/bin/twine upload -r pypitest dist/*
+
+install-test: package
+	pip uninstall -y utensor-cgen
+	pip install dist/*.tar.gz
+
+upload: package
+	.venv/bin/twine upload -r pypi dist/*
+
 clean:
-	rm -rf tests_log.txt *.pdf .pytest_cache
+	rm -rf tests_log.txt *.pdf \
+	models data \
+	tests/test_backend/{models,data} \
+	.pytest_cache dist/ build/

Pipfile.lock

@@ -72,26 +72,22 @@ Overall Architecture
 
 ::
 
-      ============       +-----------------+       ===================
-    || model file || --> | frontend Parser | --> || uTensorGraph (IR) ||
-      ============       +-----------------+       ===================
-                                                               |
-                     +-------------------------------+         |
-                     |       graph transformer       |         |
-                     | (legalization & optimization) | <------/
-                     +-------------------------------+
-                                    |
-                                    v
-                         ===========================
-                       ||       uTensorGraph        ||
-                       || (legalized and optimized) ||
-                         ===========================
-                                       |
-    +--------------------------+       |
-    | backend (code generator) | <----/
+      ============       +-----------------+       ============================
+    || model file || --> | frontend Parser | --> || uTensorGraph (IR, generic) ||
+      ============       +-----------------+       ============================
+                                                                |
+                                                                v
+                                                     +---------------------+
+                        =======================      |  graph transformer  |
+                      ||     uTensorGraph     || <-- |    (optimization)   |
+                      || (generic, optimized) ||     +---------------------+
+                        =======================                                    
+                                     |
+    +--------------------------+     |
+    | backend (code generator) | <--/
     +--------------------------+
          |
-         `---> (target files, ex: model.cpp, model.hpp, weights.idx)
+         `---> (target files, ex: model.cpp, model.hpp, weights.idx, ...etc)
 
 Basic Usage
 ===========
@@ -114,7 +110,8 @@ Convert Model File to C/C++ Code
 .. code-block:: console
 
   $ utensor-cli convert <model.pb> \
-    --output-nodes=<node_name>[,<node_name>,...]
+    --output-nodes=<node name>[,<node name>,...] \
+    [--config=config.toml]
 
 Convert given pb file into cpp/hpp files.
 
@@ -123,6 +120,8 @@ nodes you want to output, seperated by comma for multiple values.
 
 In graph theory terminology, they are ``leaf`` nodes of your graph.
 
+Use ``--config`` to pass a configuration file to the cli, you can use ``generate-config`` command to generate one (see below).
+
 example
 ~~~~~~~
 
@@ -132,8 +131,34 @@ example
 
 Run ``utensor-cli convert --help`` for detailed information.
 
-:mod:`utensor_cgen` as Library
-==============================
+Configuration
+-------------
+
+``utensor-cli`` use ``toml`` as configuration format.
+
+You can generate configuration file of given target as following:
+
+.. code-block:: console
+
+  $ utensor-cli generate-config --target <target name> [-o filename.toml]
+
+This command will generate a ``toml`` file listing all configurable values with its defaults.
+
+You can modify the value and pass the file to cli with ``--config`` flag.
+
+example
+~~~~~~~
+
+.. code-block:: console
+
+  # generate config file
+  $ utensor-cli generate-config --target utensor -o myconfig.toml
+
+  # after editting myconfig.toml
+  $ utensor-cli convert mymodel.pb --config=myconfig.toml --output-nodes=output,...
+
+Use :mod:`utensor_cgen` as Library
+==================================
 
 .. subgraph-match-begine
 
@@ -223,8 +248,8 @@ TensorFlow_
 
 1. Freeze your `tensorflow.Graph`
 
-  - please refer to the `official doc <https://www.tensorflow.org/guide/extend/model_files>`_
-    and read the `Freezing <https://www.tensorflow.org/guide/extend/model_files#freezing>`_ section
+  - please refer to this `issue track <https://github.com/tensorflow/tensorflow/issues/27614>`_ for detail
+  - especially this `comment <https://github.com/tensorflow/tensorflow/issues/27614#issuecomment-571889676>`_ by Robin2091
 
 2. Follow instructions in :ref:`install` section to install :mod:`utensor_cgen`
 

README.rst

@@ -0,0 +1,15 @@
+# https://github.com/toml-lang/toml
+# <target_name>.<component>.<part>
+[utensor.backend]
+legacy-api = true
+
+[utensor.backend.legacy_code_generator]
+src_fname = "None"
+params_dir = "data"
+embed_params_dir = "/fs/data"
+model_dir = "models"
+transform_methods = [ "dropout(name_pattern=r\"(dropout[_\\w\\d]*)/.*\")", "linear_reorder", "quantize", "conv_pool", "inline", "biasAdd", "remove_id_op", "fake_gather_v2", "refcnt",]
+save_graph = false
+debug_cmt = false
+
+[utensor.backend.graph_lower]

example_config.toml

@@ -0,0 +1,34 @@
+from textwrap import wrap
+
+from utensor_cgen.backend.base import Backend
+from utensor_cgen.backend import BackendManager
+from utensor_cgen.utils import class_property
+
+@BackendManager.register
+class DummyBackend(Backend):
+    TARGET = 'dummy-backend'
+
+    def __init__(self, config):
+        if not config:
+            config = self.default_config
+        self.output_file = config[self.TARGET][self.COMPONENT]['output-file']
+
+    def apply(self, ugraph):
+        with open(self.output_file, 'w') as fid:
+            fid.write('#include <stdio.h>\n\n')
+            fid.write('int main(int argc, char* argv[]) {\n')
+            fid.write('    printf("graph name: {}\\n");\n'.format(ugraph.name))
+            fid.write('    printf("ops in topological sorted order:\\n");\n')
+            for op_name in ugraph.topo_order:
+                fid.write('    printf("    {}\\n");\n'.format(op_name))
+            fid.write('    return 0;\n}')
+
+    @class_property
+    def default_config(cls):
+        return {
+            cls.TARGET: {
+                    cls.COMPONENT: {
+                    'output-file': 'list_op.c'
+                }
+            }
+        }

plugins/dummy_backend/__init__.py

@@ -3,27 +3,48 @@
 import os
 
 from setuptools import find_packages, setup
+from setuptools.command.develop import develop as _develop
+from setuptools.command.install import install as _install
 
 root_dir = os.path.abspath(os.path.dirname(__file__))
 with open(os.path.join(root_dir, "LICENSE")) as rf:
     license = rf.read()
 
+
+class _CompileFlatbuffMixin(object):
+
+    def run(self):
+        super(_CompileFlatbuffMixin, self).run()
+        self._build_flatbuffer()
+
+    def _build_flatbuffer(self):
+        install_dir = self.install_platlib
+        if install_dir is None:
+            install_dir = os.path.abspath('utensor')
+
+
+class _Install(_CompileFlatbuffMixin, _install): pass
+
+
+class _Develop(_CompileFlatbuffMixin, _develop): pass
+
+
 setup(
     name='utensor_cgen',
     version_config={
         "starting_version": "0.0.0",
-        "version_format": "{tag}.dev{sha:.7s}"
+        "version_format": "{tag}.{sha:.7s}.dev"
     },
     setup_requires=['better-setuptools-git-version'],
+    cmdclass={'install': _Install, 'develop': _Develop},
     description="C code generation program for uTensor",
     long_description="please go to [doc](https://utensor-cgen.readthedocs.io/en/latest/) page for more information",
-    url="https://github.com/dboyliao/utensor_cgen",
+    url="https://github.com/uTensor/utensor_cgen",
     author="Dboy Liao",
     author_email="qmalliao@gmail.com",
     license=license,
     packages=find_packages(),
-    include_package_data=True,
-    package_data={"utensor_cgen": ["backend/snippets/templates/*"]},
+    package_data={'utensor_cgen.backend.utensor.snippets': ["templates/*/*/*"]},
     entry_points={
         "console_scripts": [
             "utensor-cli=utensor_cgen.cli:cli"
@@ -38,7 +59,8 @@
         'torchvision',
         'onnx-tf==1.2.1',
         'graphviz',
-        'flatbuffers'
+        'flatbuffers',
+        'toml',
     ],
     extras_require={
         'dev': ['pytest']