Skip to content

Commit 8751b8d

Browse files
valentinedwvvalentinedwv
committed
Setup for ODM2API in PiPy.
Tests in TravisCI
1 parent 2e85eba commit 8751b8d

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+46656
-192
lines changed

‎.gitignore‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,4 +42,3 @@ log
4242
.idea
4343

4444
.DS_Store
45-
.ipynb_checkpoints

‎.travis.yml‎

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
language: python
2+
# if the https://travis-ci.org/ODM2/ODM2PythonAPI/requests ever says: missing config
3+
# validate at: http://lint.travis-ci.org/
4+
python:
5+
# - "2.6"
6+
- "2.7"
7+
# - "3.2"
8+
# - "3.3"
9+
# - "3.4"
10+
# - "3.5"
11+
# - "3.5-dev" # 3.5 development branch
12+
# - "nightly" # currently points to 3.6-dev
13+
# command to install dependencies
14+
cache:
15+
directories:
16+
- $HOME/.cache/pip/wheels
17+
- $HOME/virtualenv/python2.7.9
18+
services:
19+
- mysql
20+
- postgresql
21+
#addons:
22+
# mariadb: '10.1'
23+
before_install:
24+
# - strace mysql 2>&1 | grep cnf
25+
#stat("/etc/my.cnf", 0x7ffef7c8dca0) = -1 ENOENT (No such file or directory)
26+
#stat("/etc/mysql/my.cnf", {st_mode=S_IFREG|0644, st_size=5232, ...}) = 0
27+
#open("/etc/mysql/my.cnf", O_RDONLY|O_CLOEXEC) = 3
28+
#stat("/etc/mysql/conf.d/mariadb.cnf", {st_mode=S_IFREG|0644, st_size=435, ...}) = 0
29+
#open("/etc/mysql/conf.d/mariadb.cnf", O_RDONLY|O_CLOEXEC) = 4
30+
#stat("/etc/mysql/conf.d/mysqld_safe_syslog.cnf", {st_mode=S_IFREG|0644, st_size=36, ...}) = 0
31+
#open("/etc/mysql/conf.d/mysqld_safe_syslog.cnf", O_RDONLY|O_CLOEXEC) = 4
32+
#stat("/etc/mysql/conf.d/tokudb.cnf", {st_mode=S_IFREG|0644, st_size=285, ...}) = 0
33+
#open("/etc/mysql/conf.d/tokudb.cnf", O_RDONLY|O_CLOEXEC) = 4
34+
#stat("/home/travis/.my.cnf", 0x7ffef7c8dca0) = -1 ENOENT (No such file or directory)
35+
# - ls -l /etc/mariadb
36+
- mysqld --help --verbose
37+
- mysql -uroot -e 'SHOW VARIABLES WHERE Variable_Name LIKE "%dir"'
38+
# Default options are read from the following files in the given order:
39+
# /etc/my.cnf /etc/mysql/my.cnf ~/.my.cnf
40+
# - ls -al .my.cnf
41+
# - ls -al /etc/my.cnf
42+
- ls -al /etc/mysql/conf.d/
43+
- ls -al /etc/mysql/
44+
# - ls -al /var/lib/mysql
45+
# - ls -al /usr/local/etc/my.cnf.d/
46+
# - cat /etc/mysql/conf.d/mariadb.cnf
47+
# - cat /etc/mysql/conf.d/tokudb.cnf
48+
# - cat /etc/mysql/conf.d/mysqld_safe_syslog.cnf
49+
- cat /etc/mysql/debian-start
50+
- cat /usr/share/mysql/debian-start.inc.sh
51+
- cat /etc/init.d/mysql
52+
- sudo cat /etc/mysql/debian.cnf
53+
# - cat /etc/apparmor.d/usr.sbin.mysqld
54+
- ls -al /etc/mysql
55+
- sudo service mysql stop
56+
# - echo "[mysqld]" > /etc/my.cnf
57+
# - echo "lower_case_table_names = 1" >> /etc/my.cnf
58+
# - sudo echo "[mariadb-10.0]" >> /etc/my.cnf
59+
# - sudo echo "lower_case_table_names = 1" >> /etc/my.cnf
60+
- echo "[mysqld]" > $HOME/.my.cnf
61+
- echo "lower-case-table-names = 1" >> $HOME/.my.cnf
62+
# - echo "lower_case_table_names=1" >> $HOME/.my.cnf
63+
- cat $HOME/.my.cnf
64+
# - echo "[mariadb-10.0]" >> .my.cnf
65+
# - echo "lower_case_table_names = 1" >> .my.cnf
66+
# - sudo echo "[mysqld]" >> /etc/mysql/debian.cnf
67+
# - sudo echo "lower_case_table_names = 2" >> /etc/mysql/debian.cnf
68+
- service --status-all
69+
# - sudo cat /etc/mysql/debian.cnf
70+
# - sudo echo "[mysqld]" >> /etc/mysql/debian.cnf
71+
# - sudo echo "lower_case_table_names=1" >> /etc/mysql/debian.cnf
72+
- sudo cat /etc/mysql/debian.cnf
73+
- sudo sed -i '/\[mysqld\]/a lower_case_table_names = 1 ' /etc/mysql/my.cnf
74+
- cat /etc/mysql/my.cnf
75+
- sudo service mysql start
76+
# - sudo service mysql restart
77+
- mysql --verbose -e "show variables like 'lower%';" --user=root
78+
# - strace mysql 2>&1 | grep cnf
79+
- mysql --verbose -e "CREATE USER 'ODM'@'localhost' IDENTIFIED BY 'odm';GRANT ALL PRIVILEGES ON *.* TO 'ODM'@'localhost';" --user=root
80+
- mysql --verbose -e "CREATE USER 'ODM'@'127.0.0.1' IDENTIFIED BY 'odm';GRANT ALL PRIVILEGES ON *.* TO 'ODM'@'127.0.0.1';" --user=root
81+
- mysql --verbose -e "CREATE USER 'ODM'@'%' IDENTIFIED BY 'odm';GRANT ALL PRIVILEGES ON *.* TO 'ODM'@'%';" --user=root
82+
- mysql --verbose -e "create database IF NOT EXISTS odm2;" --user=root
83+
- mysql -e "create database IF NOT EXISTS odm2test;" --user=root
84+
- psql -U postgres -c "create extension postgis"
85+
- psql -c 'DROP DATABASE IF EXISTS odm2test;' -U postgres;
86+
- psql -c 'create database odm2test;' -U postgres;
87+
# - psql -U postgres -d odm2test -a -f ./tests/schemas/postgresql/ODM2_for_PostgreSQL.sql
88+
- psql -c 'DROP DATABASE IF EXISTS odm2;' -U postgres;
89+
- psql -c 'create database odm2;' -U postgres;
90+
# patterned after: https://github.com/ptrv/gpx2spatialite/blob/master/.travis.yml
91+
- sudo apt-get install -y python-software-properties
92+
- sudo apt-add-repository -y ppa:git-core/ppa
93+
- sudo apt-add-repository -y ppa:ubuntugis/ppa
94+
- sudo apt-get update -qq
95+
- sudo apt-get install unixodbc unixodbc-dev odbcinst1debian2 odbcinst
96+
- sudo apt-get install freetds-dev freetds-bin tdsodbc
97+
- sudo apt-get install libc6 e2fsprogs # mssql driver
98+
# see what's installed
99+
# - odbcinst -j
100+
- ls /usr/lib/x86_64-linux-gnu/odbc/
101+
- echo "[FreeTDS]"> .odbcinst.ini
102+
- echo "Description = v0.91 with protocol v7.2">> .odbcinst.ini
103+
- echo "Driver = /usr/lib/x86_64-linux-gnu/odbc/libtdsodbc.so">> .odbcinst.ini
104+
- sudo odbcinst -i -d -f .odbcinst.ini
105+
- echo "[azure]" > azure.odbc
106+
- echo "Server = nrb8xkgxaj.database.windows.net" >> azure.odbc
107+
- echo "Driver = FreeTDS" >> azure.odbc
108+
- echo "Database = odm2" >> azure.odbc
109+
- sudo odbcinst -i -s -l -f azure.odbc
110+
- echo "[moonstone]" > moonstone.odbc
111+
- echo "Server = moonstone.sdsc.edu" >> moonstone.odbc
112+
- echo "Driver = FreeTDS" >> moonstone.odbc
113+
- echo "Database = odm2" >> moonstone.odbc
114+
- sudo odbcinst -i -s -l -f moonstone.odbc
115+
# free tds
116+
- sudo echo "[FreeTDS]" >> odbcinst.ini
117+
- sudo echo "Description = v0.91 with protocol v7.2" >> odbcinst.ini
118+
- sudo echo "Driver = /usr/lib/x86_64-linux-gnu/odbc/libtdsodbc.so " >> odbcinst.ini
119+
# - sudo echo "Setup = $HOME/oob/client/libesoobsetup.so >> odbcinst.ini
120+
# - sudo echo "FileUsage = 1" >> odbcinst.ini
121+
#- ls -al /etc/odbcinst.ini /etc/odbc.ini
122+
- cat /etc/odbcinst.ini
123+
- cat /etc/odbc.ini
124+
# - ls /etc/ODBCDataSources
125+
# - cat .odbc.ini #not found
126+
#
127+
- echo "[global]" > .freetds.conf
128+
- echo "tds version = 4.2" >> .freetds.conf
129+
- echo "[kyle]" >> .freetds.conf
130+
- echo "host = nrb8xkgxaj.database.windows.net" >> .freetds.conf
131+
- echo "port = 1433" >> .freetds.conf
132+
- echo "database=odm2" >> .freetds.conf
133+
- echo "[moonstone]">> .freetds.conf
134+
- echo "host = moonstone.sdsc.edu">> .freetds.conf
135+
- echo "port = 1433">> .freetds.conf
136+
- echo "database=odm2" >> .freetds.conf
137+
- cat .freetds.conf
138+
- sudo apt-get install mysql-client
139+
# Spatialiate
140+
- sudo apt-get install -y libproj-dev libgeos-dev libspatialite-dev
141+
- sudo ln -s /usr/lib/x86_64-linux-gnu/libspatialite.so /usr/lib/libspatialite.so
142+
# - sudo apt-get install python-scipy python-matplotlib python-pandas python-sympy python-nose
143+
- sudo apt-get install python-matplotlib python-pandas python-nose
144+
- python -m pip install pandas
145+
- export PYMSSQL_BUILD_WITH_BUNDLED_FREETDS=1;python -m pip install pymssql
146+
- python -m pip install mysql-python
147+
install: # now just our code
148+
- pip install git+https://github.com/ODM2/geoalchemy.git@odm2#egg=geoalchemy-0.7.3
149+
- pip install .
150+
- pip install -r requirements_tests.txt --allow-external pyodbc --allow-unverified pyodbc
151+
# pysqlite
152+
- pip install pysqlite
153+
- pip list
154+
- ls -al ./tests/usecasesql/littlebearriver/sampledatabases/odm2_mysql/LBR_MySQL_SmallExample.sql ./tests/usecasesql/marchantariats/marchantariats.sql
155+
- mysql --user=ODM --password=odm odm2 < ./tests/usecasesql/littlebearriver/sampledatabases/odm2_mysql/LBR_MySQL_SmallExample.sql
156+
- mysql --user=root -e "show databases;"
157+
- mysql --user=root -e "select table_schema,table_name,table_rows from information_schema.tables;"
158+
- mysql --user=root -e "GRANT ALL PRIVILEGES ON odm2.* TO 'ODM'@'localhost';FLUSH PRIVILEGES;"
159+
- mysql --user=ODM --password=odm odm2 -e "use odm2; Select * from Variables;"
160+
- mysql --user=ODM --password=odm odm2 -e "use odm2; Select * from variables;"
161+
# - mysql --user=ODM --password=odm odm2 -e "Select * from odm2.Variables;"
162+
- mysql --user=ODM --password=odm -e "Select * from odm2.Variables;"
163+
- mysql --user=ODM --password=odm -e "Select * from odm2.variables;"
164+
# add -a to psql to see full log, -q is quiet
165+
- psql -U postgres -q -f ./tests/usecasesql/marchantariats/marchantariats.sql
166+
# - netstat output | grep mysql
167+
- mysql --user root -e "SELECT User, Host FROM mysql.user;"
168+
169+
# don't forget to open up the azure mssql server to these addreses
170+
# https://docs.travis-ci.com/user/ip-addresses/
171+
172+
# command to run tests
173+
script:
174+
# just the connection part
175+
- py.test tests/test_SessionFactory.py
176+
- py.test tests/test_connection.py
177+
- py.test
178+

‎appveyor.yml‎

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
environment:
2+
# patterned after: https://github.com/ogrisel/python-appveyor-demo/blob/master/appveyor.yml
3+
global:
4+
# SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the
5+
# /E:ON and /V:ON options are not enabled in the batch script intepreter
6+
# See: http://stackoverflow.com/a/13751649/163740
7+
CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\appveyor\\run_with_env.cmd"
8+
# postgres
9+
POSTGRES_PORT: tcp://localhost:5432
10+
POSTGRES_ENV_POSTGRES_USER: postgres
11+
POSTGRES_ENV_POSTGRES_PASSWORD: Password12!
12+
POSTGRES_ENV_POSTGRES_DB: odm2
13+
POSTGRES_PATH: C:\Program Files\PostgreSQL\9.4
14+
PGUSER: postgres
15+
PGPASSWORD: Password12!
16+
# mysql
17+
MYSQL_PORT: tcp://localhost:3306
18+
MYSQL_ENV_MYSQL_USER: root
19+
MYSQL_ENV_MYSQL_PASSWORD: Password12!
20+
MYSQL_ENV_MYSQL_DATABASE: odm2
21+
MYSQL_PATH: C:\Program Files\MySql\MySQL Server 5.6
22+
MYSQL_PWD: Password12!
23+
# sql server
24+
SQLSERVER_ENV_SQLSERVER_HOST: localhost
25+
SQLSERVER_ENV_SQLSERVER_PORT: 1433
26+
SQLSERVER_ENV_SQLSERVER_USER: sa
27+
SQLSERVER_ENV_SQLSERVER_PASSWORD: Password12!
28+
SQLSERVER_ENV_SQLSERVER_DATABASE: odm2
29+
matrix:
30+
31+
32+
# Pre-installed Python versions, which Appveyor may upgrade to
33+
# a later point release.
34+
# See: http://www.appveyor.com/docs/installed-software#python
35+
36+
- PYTHON: "C:\\Python27"
37+
PYTHON_VERSION: "2.7.x" # currently 2.7.9
38+
PYTHON_ARCH: "32"
39+
40+
- PYTHON: "C:\\Python27-x64"
41+
PYTHON_VERSION: "2.7.x" # currently 2.7.9
42+
PYTHON_ARCH: "64"
43+
services:
44+
- mssql2008r2sp2
45+
- mysql
46+
- postgresql
47+
48+
install:
49+
# If there is a newer build queued for the same PR, cancel this one.
50+
# The AppVeyor 'rollout builds' option is supposed to serve the same
51+
# purpose but it is problematic because it tends to cancel builds pushed
52+
# directly to master instead of just PR builds (or the converse).
53+
# credits: JuliaLang developers.
54+
- ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod `
55+
https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | `
56+
Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { `
57+
throw "There are newer queued builds for this pull request, failing early." }
58+
- ECHO "Filesystem root:"
59+
- ps: "ls \"C:/\""
60+
61+
- ECHO "Installed SDKs:"
62+
- ps: "ls \"C:/Program Files/Microsoft SDKs/Windows\""
63+
64+
# Install Python (from the official .msi of http://python.org) and pip when
65+
# not already installed.
66+
- ps: if (-not(Test-Path($env:PYTHON))) { & appveyor\install.ps1 }
67+
68+
# Prepend newly installed Python to the PATH of this build (this cannot be
69+
# done from inside the powershell script as it would require to restart
70+
# the parent CMD process).
71+
- "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
72+
# add databases
73+
- "SET PATH=%POSTGRES_PATH%\bin;%MYSQL_PATH%\bin;%PATH%"
74+
75+
# Check that we have the expected version and architecture for Python
76+
- "python --version"
77+
- "python -c \"import struct; print(struct.calcsize('P') * 8)\""
78+
79+
# Upgrade to the latest version of pip to avoid it displaying warnings
80+
# about it being out of date.
81+
- "pip install --disable-pip-version-check --user --upgrade pip"
82+
83+
# Install the build dependencies of the project. If some dependencies contain
84+
# compiled extensions and are not provided as pre-built wheel packages,
85+
# pip will build them from source using the MSVC compiler matching the
86+
# target Python version and architecture
87+
- "%CMD_IN_ENV% pip install -r dev-requirements.txt"
88+
89+
build_script:
90+
# Build the compiled extension
91+
- "%CMD_IN_ENV% python setup.py build"
92+
build_script:
93+
# Build the compiled extension
94+
- "%CMD_IN_ENV% python setup.py build"
95+
build_script:
96+
# postgres
97+
- createdb odm2
98+
- psql -d odm2 -a -f spec/databases/postgresql/schema/schema.sql
99+
# mysql
100+
- mysql -e "drop database test; create database odm2;" --user=root
101+
- mysql sqlectron < spec/databases/mysql/schema/schema.sql --user=root
102+
# sqlserver
103+
- ps: ./appveyor-sqlserver.ps1 SQL2008R2SP2
104+
- sqlcmd -S localhost,1433 -U sa -P Password12! -Q "CREATE DATABASE odm2" -d "master"
105+
- sqlcmd -S localhost,1433 -U sa -P Password12! -i spec/databases/sqlserver/schema/schema.sql -d "odm2"
106+
- sqlcmd -S localhost,1433 -U sa -P Password12! -Q "select table_name from information_schema.tables" -d "odm2"
107+
108+
test_script:
109+
# Run the project tests
110+
- "%CMD_IN_ENV% python setup.py nosetests"
111+
112+
language: python
113+
python:
114+
# - "2.6"
115+
- "2.7"
116+
# - "3.2"
117+
# - "3.3"
118+
# - "3.4"
119+
# - "3.5"
120+
# - "3.5-dev" # 3.5 development branch
121+
# - "nightly" # currently points to 3.6-dev
122+
# command to install dependencies
123+
124+
before_install:
125+
#https://github.com/sqlectron/sqlectron-core/blob/master/appveyor.yml
126+
- mysql -e "CREATE USER 'ODM'@'localhost' IDENTIFIED BY 'odm';GRANT ALL PRIVILEGES ON *.* TO 'ODM'@'localhost';" -uroot
127+
- mysql -e "create database IF NOT EXISTS odm2;" -uroot
128+
- mysql -e "create database IF NOT EXISTS odm2test;" -uroot
129+
- psql -U postgres -c "create extension postgis"
130+
- psql -c 'DROP DATABASE IF EXISTS odm2test;' -U postgres;
131+
- psql -c 'create database odm2test;' -U postgres;
132+
# - psql -U postgres -d odm2test -a -f ./tests/schemas/postgresql/ODM2_for_PostgreSQL.sql
133+
- psql -c 'DROP DATABASE IF EXISTS odm2;' -U postgres;
134+
- psql -c 'create database odm2;' -U postgres;
135+
# patterned after: https://github.com/ptrv/gpx2spatialite/blob/master/.travis.yml
136+
# - sudo apt-get install -y python-software-properties
137+
# - sudo apt-add-repository -y ppa:git-core/ppa
138+
# - sudo apt-add-repository -y ppa:ubuntugis/ppa
139+
# - sudo apt-get update -qq
140+
# - sudo apt-get install unixodbc unixodbc-dev tdsodbc
141+
# - sudo apt-get install freetds-dev freetds-bin
142+
# - sudo apt-get install libc6 e2fsprogs # mssql driver
143+
# Spatialiate
144+
# - sudo apt-get install -y libproj-dev libgeos-dev libspatialite-dev
145+
# - sudo ln -s /usr/lib/x86_64-linux-gnu/libspatialite.so /usr/lib/libspatialite.so
146+
# - sudo apt-get install python-scipy python-matplotlib python-pandas python-sympy python-nose
147+
# - sudo apt-get install python-matplotlib python-pandas python-nose
148+
- pip install pandas
149+
install: # now just our code
150+
- pip install git+https://github.com/ODM2/geoalchemy.git@odm2#egg=geoalchemy-0.7.3
151+
- pip install .
152+
- pip install -r requirements_tests.txt --allow-external pyodbc --allow-unverified pyodbc
153+
# pysqlite
154+
- pip install pysqlite
155+
- dir .\tests\usecasesql\littlebearriver\sampledatabases\odm2_mysql\LBR_MySQL_SmallExample.sql .\tests\usecasesql\marchantariats\marchantariats.sql
156+
- mysql --user root --verbose odm2 < .\tests\usecasesql\littlebearriver\sampledatabases\odm2_mysql\LBR_MySQL_SmallExample.sql
157+
# add -a to psql to see full log
158+
- psql -U postgres -f ./tests/usecasesql/marchantariats/marchantariats.sql
159+
160+
# don't forget to open up the azure mssql server to these addreses
161+
# https://docs.travis-ci.com/user/ip-addresses/
162+
163+
# command to run tests
164+
script:
165+
# just the connection part
166+
- py.test tests/test_connection.py
167+
- py.test
168+

‎distribution_notes.txt‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
2+
Testing a package
3+
* upload to pyptest: https://wiki.python.org/moin/TestPyPI
4+
https://testpypi.python.org/pypi
5+
6+
python setup.py sdist upload -r https://testpypi.python.org/pypi
7+
8+
=== testing
9+
cd dev_odm
10+
virtualenv --system-site-packages v_odmtest
11+
cd ODM2PythonAPI
12+
python setup.py install
13+
# above failed on windows... need to be able to compile code
14+
15+
=== CONDA == WINDOWS
16+
Start anaconda console
17+
conda search "^python$"
18+
conda create -n v_odm2test python=2.7.11 anaconda panadas sqlalchemy six geos pyodbc
19+
activate v_odm2test
20+
conda install -c SciTools shapely # to get binary versions
21+
22+
23+
conda remove --name v_odm2test --all
24+
# note issue solved by editing (anacona)\Scripts\activate.bat
25+
http://stackoverflow.com/questions/34818282/anaconda-prompt-loading-error-the-input-line-is-too-long
26+
===
27+
note for packaging yodatools
28+
https://gehrcke.de/2014/02/distributing-a-python-command-line-application/
29+
30+
creating the databases:
31+
connection string for spatiallite
32+
sqlite+pysqlite:///../../../ODM2PythonAPI/tests/spatialite/odm2_test.sqlite

‎notes_sqlacodegen.txt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
2+
https://pypi.python.org/pypi/sqlacodegen
3+
4+
pip install sqlacodegen
5+
6+
sqlacodegen "mssql+pyodbc://user:password@localhost/ODM2?driver=SQL+Server+Native+Client+10.0"
7+
8+
sqlacodegen "mssql+pyodbc://user:password@localhost/ODM2_201512?driver=SQL+Server+Native+Client+10.0" --schema ODM2 --outfile generated_schema_201512.py

0 commit comments

Comments
 (0)