Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add provenance and index database management #13

Open
wants to merge 46 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
46 commits
Select commit Hold shift + click to select a range
9a65bfe
added uid in Factory
revesansparole Jul 20, 2016
2aa8f76
added uid to factories and cleaned wraleas
revesansparole Jul 20, 2016
7d9d456
put provenance back into the system
revesansparole Jul 26, 2016
e076241
added get method in metadatadict
revesansparole Jul 27, 2016
3ebaa0c
removed bad init of provenance
revesansparole Jul 27, 2016
ef89307
bug correction call to after_eval
revesansparole Jul 27, 2016
2ee2560
preserve uid over rewriting of cnf
revesansparole Jul 27, 2016
513c827
preserve uid over rewriting, bug correction when writing node factories
revesansparole Jul 27, 2016
a4aaf02
added IRef interface
revesansparole Aug 4, 2016
475e817
added openalea.provenance as requirement
revesansparole Aug 4, 2016
0b885b5
add *args to BrutEval so it doesnt fail with unexpected None arguments
pomme-abricot Oct 4, 2019
810e516
add *args to BrutEval so it doesnt fail with unexpected None arguments
pomme-abricot Oct 4, 2019
c76e34e
add *args to BrutEval so it doesnt fail with unexpected None arguments
pomme-abricot Oct 4, 2019
d3d5c80
add record_provenance as parameter for cn.eval so it can get the prov…
pomme-abricot Oct 4, 2019
b5530d8
change uid def to be the name - same at each construction
pomme-abricot Oct 7, 2019
5aea54a
enable linting banit
pomme-abricot Dec 5, 2019
52b3e20
add misc functions use to capture provenance
pomme-abricot Dec 5, 2019
1994b71
Add CacheIndex & Prov class
pomme-abricot Dec 5, 2019
dd53673
some function in tests
pomme-abricot Dec 5, 2019
75953a5
add an eval that use costs
pomme-abricot Dec 5, 2019
78010a6
BrutEval can save provenance into file
pomme-abricot Dec 5, 2019
e15b563
ignore visual studio param
pomme-abricot Dec 5, 2019
8a0b95f
start using TetsEval for provenacne tracking tests
pomme-abricot Dec 5, 2019
d98286e
Add evaluator that evaluate wf fragments
pomme-abricot Dec 11, 2019
05668d0
Write and load data with dill
pomme-abricot Dec 11, 2019
6f82a0b
remove old functions
pomme-abricot Dec 12, 2019
27e23b3
add cassandra workbench
pomme-abricot Dec 12, 2019
fedb427
add vscode paramters
pomme-abricot Dec 16, 2019
79c8754
.
pomme-abricot Dec 16, 2019
91765eb
use global pahts
pomme-abricot Dec 16, 2019
9992069
add zmq evaluation
pomme-abricot Dec 16, 2019
d3f426a
add mongo as provenance database
pomme-abricot Dec 16, 2019
ac20b8b
.
pomme-abricot Dec 17, 2019
0eca1cc
update for fragment eval + cassandra
pomme-abricot Dec 18, 2019
38c336e
.
pomme-abricot Dec 19, 2019
e88a28f
refactor - put source in distributed pkg
pomme-abricot Jan 13, 2020
1520b99
add path for writing tmp data
pomme-abricot Jan 14, 2020
e2eee08
fake eval that print ports id
pomme-abricot Jan 15, 2020
7f23bee
fix names
pomme-abricot Jan 17, 2020
182487b
Provenance works with Files, Mongo
pomme-abricot Jan 20, 2020
5a57be2
add cassandra provenance
pomme-abricot Jan 20, 2020
6b784b3
data index working
pomme-abricot Jan 21, 2020
1530714
remove provenance_parameter in eval
pomme-abricot Jan 22, 2020
150e296
FakeEval is generating tasks UIDs
pomme-abricot Jan 29, 2020
3eba343
Merge remote-tracking branch 'upstream/master' into prov
pomme-abricot Jan 30, 2020
842554d
remove test evaluation
pomme-abricot Jan 31, 2020
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 68 additions & 37 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,6 +1,46 @@
# Python files
# {# pkglts, github
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]

# pkglts files
.pkglts/info.log*

# Packages
*.egg
*.egg-info
.eggs
.Python
*.pth
dist/
build/
env/
downloads/
eggs/
parts/
bin/
var/
sdist/
develop-eggs/
.installed.cfg
lib/
lib64/

# editors
.idea/
.vscode/

# Vim files
*.swp
*.*~

# Mr Developer
.mr.developer.cfg
.project
.pydevproject
.settings


# C extensions
*.so
*.dll
Expand All @@ -14,54 +54,45 @@
# Compiled Object files
*.os

# Packages
*.egg
*.egg-info
dist
build
build-scons
eggs
parts
bin
var
sdist
develop-eggs
.installed.cfg
lib
lib64
__pycache__
*.pth
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec

# Installer logs
pip-log.txt
pip-delete-this-directory.txt
.amlog
.sconsign.dblite

# Unit test / coverage reports
.coverage
.tox
nosetests.xml

# Designer files
*visualea/src/visualea/ui_*

# Translations
*.mo
*.pot

# Vim files
*.swp
*.*~
# Django stuff:
*.log

# Mr Developer
.mr.developer.cfg
.project
.pydevproject
.settings
.idea
# PyBuilder
target/

# jupyter notebooks
.ipynb_checkpoints/

# svn
.svn

# temporary files
*._icon.png

# coverage
.coverage

# sphinx autogen file
doc/_dvlpt/


# #}

# user custom filters
# ignore cassandra files
.cassandraWorkbench*
.vscode/
90 changes: 37 additions & 53 deletions .pkglts/pkg_cfg.json
Original file line number Diff line number Diff line change
@@ -1,81 +1,65 @@
{
"_pkglts": {
"auto_install": true,
"install_front_end": "stdout",
"use_prompts": false,
"version": 11
},
"auto_install": true,
"install_front_end": "stdout",
"use_prompts": false,
"version": 3
},
"base": {
"authors": [
[
"Christophe Pradal",
"Christophe Pradal",
"christophe dot pradal at cirad dot fr"
],
],
[
"Samuel Dufour-Kowalski",
"Samuel Dufour-Kowalski",
"dufourko at cirad dot fr"
],
],
[
"revesansparole",
"revesansparole",
"[email protected]"
],
[
"Guillaume Baty",
""
]
],
"namespace": "openalea",
"namespace_method": "pkg_util",
"pkgname": "core",
"url": "https://github.com/openalea/core"
},
"coverage": {},
],
"namespace": "openalea",
"pkgname": "core",
"url": null
},
"coverage": {},
"doc": {
"description": "OpenAlea.Core is able to discover and manage packages and logical components, build and evaluate dataflows and Generate final applications",
"fmt": "rst",
"description": "OpenAlea.Core is able to discover and manage packages and logical components, build and evaluate dataflows and Generate final applications",
"keywords": [
"openalea"
],
"contributors": "OpenAlea",
"authors": "Christophe Pradal"
},
"git": {},
]
},
"github": {
"owner": "{{ base.authors[0][0] }}",
"project": "{{ base.pkgname }}",
"owner": "openalea",
"project": "{{ base.pkgname }}",
"url": "https://github.com/{{ github.owner }}/{{ github.project }}"
},
},
"license": {
"name": "cecill-c",
"organization": "CIRAD/INRIA",
"project": "{{ base.pkgname }}",
"name": "cecill-c",
"organization": "CIRAD/INRIA",
"project": "{{ base.pkgname }}",
"year": 2015
},
},
"pysetup": {
"intended_versions": [
"27"
],
"require": [
{
"name": "openalea/deploy",
"pkg_mng": "git"
}
]
},
],
"require": [["git", "openalea/deploy"], ["git", "openalea/provenance"]]
},
"readthedocs": {
"project": "openalea-core"
},
},
"sphinx": {
"autodoc_dvlpt": true,
"build_dir": "build/sphinx",
"autodoc_dvlpt": true,
"theme": "default"
},
"test": {
"suite_name": "pytest"
},
},
"test": {},
"travis": {},
"version": {
"major": 2,
"minor": 0,
"post": 2
"major": 1,
"minor": 4,
"post": 0
}
}
82 changes: 41 additions & 41 deletions .pkglts/pkg_hash.json
Original file line number Diff line number Diff line change
@@ -1,66 +1,66 @@
{
".coveragerc": {
"coverage": "fJOlJPmBiCjF4ydI8eATEwZLJXAhQR/S/fMnVio8FMbz4iTTM+1JD+8J0wJ+Je7t0Fi7sY+cxlRjpumCigbtVg=="
},
},
".gitignore": {
"git": "BajiufpLax6rf5dmemG7nvqJTLIyPShHtj2YTKyJ7vgPbEZrmsUMebpVX4WoruGBcRQwPOqS5fn5D/PjPmQErw=="
},
"github": "wVVlsQPP65I5Gg6bW37JENFtYxO6Naq7x+mO0DaDGFVbMPk2o5QJ3DUwMoYfkOfou9lcR/6DC0MXr7hnrTWc9g=="
},
".travis.yml": {
"travis": "p39laCS5PHN7gbwAtgU28Z+UXHRifGIORlali3ZkK7OQKjjRhn1Dbwscp76SZeocjKUEBtl3fI1857Ri37wPWQ==",
"travis.addons": "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==",
"travis.after": "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==",
"travis.install": "uieq8tZxbKnpbCZFJeBMPIUo7cheWgDZyiVg50+CQR2yN9/eejR+Go6jNh+0S89LtB5LgmlQ0Ao3KTB3xaN8Xg==",
"travis.script": "DuMVu20DK3x0WVdVZxihdiYnEkfDqmOlMUuCHohyYvi4fv3Ew8OeAca6AdKM/ZCSZvVS9tywx+PhPrmDexl6hg=="
},
"travis": "p39laCS5PHN7gbwAtgU28Z+UXHRifGIORlali3ZkK7OQKjjRhn1Dbwscp76SZeocjKUEBtl3fI1857Ri37wPWQ==",
"travis.addons": "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==",
"travis.after": "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==",
"travis.install": "Ay/BrD6JoeZi+JszHb0sHpFfzVEp9qaeFsWBBBJG0t2xwgF3tinQxifkNFwuJdlyCjy6U9k901dZnqt6yrzzxQ==",
"travis.script": "STjpu3u3YoOeTz4mBIflZ58OodPdFRPFtLt6c8CUciNwe/fYhCTxlKTMB/S7VL1ippUsQVIfrFGl+OQ/ALWmPQ=="
},
"AUTHORS.rst": {
"doc": "mr4dDvb808KCdcLSRWe9brxcIBamJp3DMpFzoEhHCeSLmr7Q0QfrNRZ79dnneEyWHgcs2srtibLIZlGOq1S0rQ=="
},
"CONTRIBUTING.rst": {},
"HISTORY.rst": {},
"doc": "HvXVe1vS1t/xzXuerP756vvyf3I4V8xEOqvNIEGejKpuzVRZr6wR4C24rm8O9OnBsYedYHvDxQRF25urtQCDMQ=="
},
"CONTRIBUTING.rst": {},
"HISTORY.rst": {},
"LICENSE": {
"license": "21ExMhSzB2uXQuU1VoH7++nrb4u6TbesqGtJDpM39OLPWft9SJ2tJ956R6foYmkI9VuJay7SUrhFLPxxMEHLxg=="
},
},
"MANIFEST.in": {
"pysetup": "vzgcfdwg7fafu0G3HEmFsZnC8jB8sa4zeSFQ2ReZO+pUfGeynOcw4+CIzP/ZV7n5nmpGGI7xqcdcPtOFfoivdQ=="
},
},
"README.rst": {
"doc": "TgYu4NHXwUZvrQ1Qz0L2XXLNIO5XdbfOzfd9SLpaFYHkcByYGk0nzosJrGveh1tzoGHf8mHbIh3j4hJuZA1YAQ=="
},
"doc": "324FZvzs5aJAtcREXT4nIIv/zjrfc0EZj+qjxyrZ9EP+UXQERzKyDx+4u1FDMMfzP2WJhOkPf9QQXzV4T+92GA=="
},
"doc/Makefile": {
"sphinx": "C58jxdu8XJNOoaWOw7VWy4LwFmS8lTQt/JeZLt5JozMSJ503plOZchucx+vAEzELbx/Czs/h9pshe9s+9HeOeQ=="
},
"doc/_static/nonempty.txt": {},
"doc/authors.rst": {},
},
"doc/_static/nonempty.txt": {},
"doc/authors.rst": {},
"doc/conf.py": {
"sphinx": "2tQ8D7EDdBCKug4c/mp+T6BWPLejyfJxv7mvT+e+YeHWxWEf+x3rqpFdHax/2g7Kx6mFSJJjD4OZtD3SFQ0sug=="
},
"doc/contributing.rst": {},
"doc/history.rst": {},
"doc/index.rst": {},
"doc/installation.rst": {},
"sphinx": "vfmn2Bw5s7XSpWirr5eXEjxifmpqNg4YCiNCojKEmRaCHDw3F/yzd6ghqanWEdjEkbs7Dn9Z9U3jdHBM6/j3oA=="
},
"doc/contributing.rst": {},
"doc/history.rst": {},
"doc/index.rst": {},
"doc/installation.rst": {},
"doc/make.bat": {
"sphinx": "oA67PK7JSR3w9u1V0gOXFZAb8qgfgUJ0wQ46cVXu7M3iJyAbLf76WkEqw6Y1r5Q6EvHAogcjmImdGsOpiYZ22g=="
},
"doc/readme.rst": {},
"doc/usage.rst": {},
},
"doc/readme.rst": {},
"doc/usage.rst": {},
"dvlpt_requirements.txt": {
"pysetup": "ONHd7KJUHI6bXimIhJ5wQNBLkFxjrrx8QaUiAWX5VuehOEm8oScMVo6MBE1ir0kydnZQMPfVfE2xEsCIx0OQKA=="
},
"pysetup": "XWPKeIfT2RoNPeDhPDgWWOssKAjkq/OXoEfIpSs8vJrYkPahRrEs1qXFvqxzqoV09MjmnVg0zrG9m/wRXfrRxw=="
},
"requirements.txt": {
"pysetup": "Wv2Gn+Cy4Ukfy3XXDV3p9fubX0FVqMuS8E22TTvSMcFdhxdVqQIZP6vTgFVAFbdZ4MDsP9xN7IVXQr177j00hQ=="
},
"pysetup": "ROpjsUQMjMbn87UPzWB8XD7JnQAUNptjTBj99UwCmDUKNGQNqmlZD0gN+ukVdk4E87brwVEgqOZ47nid1EGpww=="
},
"setup.cfg": {
"pysetup": "60E8EHReCNkZG5fVvzln5jA58snyNhSP2LiCl0ngVMVo9R1qT6G/JRnGQpx8+c6trba9O5teM0eZ4PUNXh/QEw=="
},
"pysetup": "QBedkKDNLW8WmznOFdyIyCL2sTDHYcmudSzjx+7QUcRI1vVxIHnNm82aiMvaMeEAYpOCr+8+Nj4ZTr9P0dSxSg=="
},
"setup.py": {
"pysetup.call": "2wHuP94SpWKZpqOzExkQGTxnhGPgBHK3qOdPn0Ncstj5FkZ7FBGGM/inqAXaI4QLKQtTNlOUYRGSkYUWAUcf1w==",
"pysetup.kwds": "lvC2rA9j9gbWQdPVGNGAhw6qs2S8QktXHUGLpZ4GhykYRbGFqqlC9B9Wdzktgqspie28hxjP+yCQorsxi38PUg=="
},
"pysetup.call": "2wHuP94SpWKZpqOzExkQGTxnhGPgBHK3qOdPn0Ncstj5FkZ7FBGGM/inqAXaI4QLKQtTNlOUYRGSkYUWAUcf1w==",
"pysetup.kwds": "dlgrvck1HbacSgA60B/51tvoA65q1tPJ/9ZWW0mrNh0uCq4ceY5GW2McnmDVal/PjP19xCfhoyUBuElz9HvuSA=="
},
"src/openalea/core/__init__.py": {
"base": "gR33dW0qqYmsV9NSNB+DD8XmuxnC2t0mKjnMoU5728qh97fSER6MbX+3QKxpZDLByZToaAay4xhx8acxketJmA=="
},
},
"src/openalea/core/version.py": {
"version": "xMbInCdDTk9EH4B/UZg9ywcImWAvphx9GNRPd1pqfQuDuBxOkF60Y3if3LGty+BFNAXheBp782+M8vMWyY2Uqw=="
},
"version": "uGFlizzcSxlY4JEVKPFfVPXszFKJiUw9c8Qi6ykA2LAe63uOU3NiKVlpwCmmmHHR+TSxwUnoSGLg+QuZ5yV//g=="
},
"test/__init__.py": {}
}
1 change: 0 additions & 1 deletion .travis.yml

This file was deleted.

Loading