Skip to content

Commit e4151aa

Browse files
zurkzurk
authored
Merge pull request #3 from Guillemdb/ci
Add Ci and make it pass
2 parents 718c237 + 6731bea commit e4151aa

13 files changed

Lines changed: 376 additions & 0 deletions

‎.coveragerc‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
2+
[run]
3+
branch = True
4+
source = sourced/ml/core
5+
6+
[report]
7+
exclude_lines =
8+
no cover
9+
raise NotImplementedError
10+
if __name__ == "__main__":
11+
ignore_errors = True
12+
omit =
13+
sourced/ml/core/tests/*
14+
sourced/ml/core/swivel.py
15+
sourced/ml/core/bigartm.py

‎.flake8‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
[flake8]
2+
ignore=B008,E121,E123,E126,E203,E226,E24,E704,W503,W504,D100,D105,D200,D301,D402
3+
max-line-length=99
4+
exclude=
5+
.git
6+
doc
7+
inline-quotes="
8+
import-order-style=appnexus
9+
application-package-names=sourced.ml.core
10+
per-file-ignores=
11+
**/tests/**:D
12+
# Should be resolved one by one
13+
# Related issue: https://github.com/src-d/ml/issues/354
14+
./sourced/ml/core/extractors/*:D
15+
./sourced/ml/core/models/**:D
16+
./sourced/ml/core/algorithms/**:D
17+
./sourced/ml/core/utils/*:D

‎.gitignore‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
2+
#Mac OS
3+
*.DS_Store
4+
5+
#PyCharm IDE
6+
.idea/
7+
8+
# Documentation build files
9+
doc/_build/
10+
doc/ast2vec.rst
11+
doc/modules.rst
12+
13+
# Byte-compiled / optimized / DLL files
14+
__pycache__/
15+
*.py[cod]
16+
*$py.class
17+
18+
# C extensions
19+
*.so
20+
21+
# Distribution / packaging
22+
.Python
23+
env/
24+
build/
25+
develop-eggs/
26+
dist/
27+
downloads/
28+
eggs/
29+
.eggs/
30+
lib/
31+
lib64/
32+
parts/
33+
sdist/
34+
var/
35+
wheels/
36+
*.egg-info/
37+
.installed.cfg
38+
*.egg
39+
40+
# PyInstaller
41+
# Usually these files are written by a python script from a template
42+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
43+
*.manifest
44+
*.spec
45+
46+
# Installer logs
47+
pip-log.txt
48+
pip-delete-this-directory.txt
49+
50+
# Unit test / coverage reports
51+
htmlcov/
52+
.tox/
53+
.coverage
54+
.coverage.*
55+
.cache
56+
nosetests.xml
57+
coverage.xml
58+
*.cover
59+
.hypothesis/
60+
61+
# Translations
62+
*.mo
63+
*.pot
64+
65+
# Django stuff:
66+
*.log
67+
local_settings.py
68+
69+
# Flask stuff:
70+
instance/
71+
.webassets-cache
72+
73+
# Scrapy stuff:
74+
.scrapy
75+
76+
# Sphinx documentation
77+
docs/_build/
78+
79+
# PyBuilder
80+
target/
81+
82+
# Jupyter Notebook
83+
.ipynb_checkpoints
84+
85+
# pyenv
86+
.python-version
87+
88+
# celery beat schedule file
89+
celerybeat-schedule
90+
91+
# SageMath parsed files
92+
*.sage.py
93+
94+
# dotenv
95+
.env
96+
97+
# virtualenv
98+
.venv
99+
venv/
100+
ENV/
101+
102+
# Spyder project settings
103+
.spyderproject
104+
.spyproject
105+
106+
# Rope project settings
107+
.ropeproject
108+
109+
# mkdocs documentation
110+
/site
111+
112+
# mypy
113+
.mypy_cache/
114+
115+
# CI
116+
.ci

‎.pylintrc‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
[MASTER]
2+
jobs=0
3+
load-plugins=pylint.extensions.docparams
4+
5+
[MESSAGES CONTROL]
6+
disable=all
7+
enable=missing-param-doc,
8+
differing-param-doc,
9+
differing-type-doc,
10+
missing-return-doc

‎.travis.yml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
language: python
2+
sudo: true
3+
dist: xenial
4+
services:
5+
- docker
6+
cache: pip
7+
before_cache:
8+
- chown -R travis:travis $HOME/.cache/pip
9+
stages:
10+
- style
11+
- test
12+
_install: &_install
13+
- travis_retry make bblfsh-start
14+
- pip install --upgrade pip cython codecov
15+
- ML_CORE_SETUP_INCLUDE_TESTS=1 pip install .[tf]
16+
- cd $(pip show sourced.ml.core|grep Location|cut -d' ' -f2)/sourced/ml/core
17+
- find . -wholename "*/tests/*" -type d -exec chmod 555 {} \;
18+
_coverage: &_coverage
19+
- coverage run --concurrency=multiprocessing -m unittest discover
20+
- travis_retry coverage combine
21+
matrix:
22+
fast_finish: true
23+
include:
24+
- stage: style
25+
python: 3.7
26+
script:
27+
- make check
28+
install:
29+
- pip install -r requirements-lint.txt
30+
- stage: test
31+
python: 3.5
32+
script: *_coverage
33+
install: *_install
34+
- stage: test
35+
python: 3.6
36+
script: *_coverage
37+
install: *_install
38+
- stage: test
39+
python: 3.7
40+
script: *_coverage
41+
install: *_install
42+
after_success:
43+
- codecov
44+
- stage: test
45+
name: Tests inside docker
46+
script:
47+
- make docker-build VERSION=test
48+
- make docker-test VERSION=test
49+
install:
50+
- travis_retry make bblfsh-start
51+
notifications:
52+
email: false

‎Dockerfile‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
FROM ubuntu:18.04
2+
3+
ENV BROWSER=/browser \
4+
LC_ALL=en_US.UTF-8
5+
6+
COPY requirements.txt ml_core/requirements.txt
7+
8+
RUN apt-get update && \
9+
apt-get install -y --no-install-suggests --no-install-recommends \
10+
ca-certificates locales libxml2 libxml2-dev gcc g++ wget \
11+
python3 python3-dev python3-distutils && \
12+
echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
13+
locale-gen && \
14+
wget -O - https://bootstrap.pypa.io/get-pip.py | python3 && \
15+
cd ml_core && \
16+
pip3 install --no-cache-dir -r requirements.txt && \
17+
apt-get remove -y python3-dev libxml2-dev gcc g++ wget && \
18+
apt-get remove -y .*-doc .*-man >/dev/null && \
19+
apt-get autoremove -y && \
20+
apt-get clean && \
21+
rm -rf /var/lib/apt/lists/* && \
22+
echo '#!/bin/bash\n\
23+
\n\
24+
echo\n\
25+
echo " $@"\n\
26+
echo\n\' > /browser && \
27+
chmod +x /browser
28+
29+
COPY . ml_core/
30+
RUN cd ml_core && pip3 install -e .

‎Makefile‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
current_dir = $(shell pwd)
2+
3+
PROJECT = ml_core
4+
5+
DOCKERFILES = Dockerfile:$(PROJECT)
6+
DOCKER_ORG = "srcd"
7+
8+
# Including ci Makefile
9+
CI_REPOSITORY ?= https://github.com/src-d/ci.git
10+
CI_BRANCH ?= v1
11+
CI_PATH ?= .ci
12+
MAKEFILE := $(CI_PATH)/Makefile.main
13+
$(MAKEFILE):
14+
git clone --quiet --depth 1 -b $(CI_BRANCH) $(CI_REPOSITORY) $(CI_PATH);
15+
-include $(MAKEFILE)
16+
17+
.PHONY: check
18+
check:
19+
! (grep -R /tmp sourced/ml/core/tests)
20+
flake8 --count
21+
pylint sourced
22+
23+
.PHONY: test
24+
test:
25+
python3 -m unittest discover
26+
27+
.PHONY: docker-test
28+
docker-test:
29+
docker ps | grep bblfshd # bblfsh server should be run. Try `make bblfsh-start` command.
30+
docker run --rm -it --network host --entrypoint python3 -w /ml_core \
31+
-e SKIP_BBLFSH_UTILS_TESTS=1 \
32+
srcd/ml_core:$(VERSION) -m unittest discover
33+
34+
.PHONY: bblfsh-start
35+
bblfsh-start:
36+
! docker ps | grep bblfshd # bblfsh server should not be running already
37+
docker run -d --name ml_core_bblfshd --privileged -p 9432\:9432 bblfsh/bblfshd\:v2.12.1
38+
docker exec -it ml_core_bblfshd bblfshctl driver install python bblfsh/python-driver\:v2.9.0

‎requirements-lint.txt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
flake8==3.5.0
2+
flake8-bugbear==18.8.0
3+
flake8-docstrings==1.3.0
4+
flake8-import-order==0.18.1
5+
flake8-quotes==1.0.0
6+
flake8-per-file-ignores==0.8.1
7+
pylint==2.3.1

‎requirements.txt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
Cython>=0.28,<1.0; python_version == '3.7'
2+
PyStemmer==1.3.0
3+
bblfsh>=2.12.7,<3.0
4+
modelforge==0.12.1
5+
pygments==2.3.1
6+
keras==2.2.4
7+
scikit-learn==0.20.3
8+
tqdm==4.31.1

‎setup.py‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
from importlib.machinery import SourceFileLoader
2+
import io
3+
import os.path
4+
5+
from setuptools import find_packages, setup
6+
7+
sourcedml = SourceFileLoader("sourced-ml-core", "./sourced/ml/core/__init__.py").load_module()
8+
9+
with io.open(os.path.join(os.path.dirname(__file__), "README.md"), encoding="utf-8") as f:
10+
long_description = f.read()
11+
12+
tf_requires = ["tensorflow>=1.0,<2.0"]
13+
tf_gpu_requires = ["tensorflow-gpu>=1.0,<2.0"]
14+
exclude_packages = (
15+
("sourced.ml.core.tests", "sourced.ml.core.tests.source")
16+
if not os.getenv("ML_CORE_SETUP_INCLUDE_TESTS", False)
17+
else ()
18+
)
19+
20+
21+
setup(
22+
name="sourced-ml-core",
23+
description="Library containing the core algorithms for machine learning on source code. "
24+
"Provides API and tools to train and use models based "
25+
"on source code features extracted from Babelfish's UASTs.",
26+
long_description=long_description,
27+
long_description_content_type="text/markdown",
28+
version=sourcedml.__version__,
29+
license="Apache 2.0",
30+
author="source{d}",
31+
author_email="machine-learning@sourced.tech",
32+
url="https://github.com/src-d/ml-core",
33+
download_url="https://github.com/src-d/ml-core",
34+
packages=find_packages(exclude=exclude_packages),
35+
keywords=[
36+
"machine learning on source code",
37+
"word2vec",
38+
"id2vec",
39+
"github",
40+
"swivel",
41+
"bow",
42+
"bblfsh",
43+
"babelfish",
44+
],
45+
install_requires=[
46+
"PyStemmer>=1.3,<2.0",
47+
"bblfsh>=2.12.7,<3.0",
48+
"modelforge>=0.12.1,<0.13",
49+
"pygments>=2.2.0,<3.0",
50+
"keras>=2.0,<3.0",
51+
"scikit-learn>=0.19,<1.0",
52+
"tqdm>=4.20,<5.0",
53+
],
54+
extras_require={"tf": tf_requires, "tf_gpu": tf_gpu_requires},
55+
tests_require=["docker>=3.6.0,<4.0"],
56+
package_data={
57+
"": ["LICENSE.md", "README.md"],
58+
"sourced.ml.core.tests": ["./asdf/*.asdf", "./swivel/*", "identifiers.csv.tar.gz"],
59+
},
60+
python_requires=">=3.5",
61+
classifiers=[
62+
"Development Status :: 3 - Alpha",
63+
"Environment :: Console",
64+
"Intended Audience :: Developers",
65+
"License :: OSI Approved :: Apache Software License",
66+
"Operating System :: POSIX",
67+
"Programming Language :: Python :: 3.5",
68+
"Programming Language :: Python :: 3.6",
69+
"Programming Language :: Python :: 3.7",
70+
"Topic :: Software Development :: Libraries",
71+
],
72+
)

0 commit comments

Comments
 (0)