Merge pull request #46 from ODM2/setup_clean

Setup clean

Author: valentinedwv

.gitignore

@@ -1,5 +1,6 @@
 *.py[cod]
 *~
+.ipynb_checkpoints
 
 #  C extensions
 *.so
@@ -42,4 +43,3 @@ log
 .idea
 
 .DS_Store
-.ipynb_checkpoints

.travis.yml

@@ -0,0 +1,65 @@
+language: python
+sudo: required
+# if the https://travis-ci.org/ODM2/ODM2PythonAPI/requests ever says: missing config
+# validate at: http://lint.travis-ci.org/
+python:
+#  - "2.6"
+  - "2.7"
+#  - "3.2"
+#  - "3.3"
+#  - "3.4"
+#  - "3.5"
+#  - "3.5-dev" # 3.5 development branch
+#  - "nightly" # currently points to 3.6-dev
+# command to install dependencies
+cache:
+  directories:
+    - $HOME/.cache/pip/wheels
+    - $HOME/virtualenv/python2.7.9
+services:
+  - mysql
+  - postgresql
+addons:
+  apt:
+    packages:
+      - cmake
+      - unixodbc
+      - unixodbc-dev
+      - odbcinst1debian2
+      - odbcinst
+      - freetds-dev
+      - freetds-bin
+      - tdsodbc
+      - libc6
+      - e2fsprogs
+      - mysql-client
+      - libproj-dev
+      - libgeos-dev
+      - libspatialite-dev
+#  mariadb: '10.1'
+before_script:
+  - ./scripts/mysql_setup.sh
+  - ./scripts/postgres_setup.sh
+  - ./scripts/freetds.sh
+
+before_install:
+# python -m pip makes the install go into the virtualenv
+  - python -m pip install pandas
+  - export PYMSSQL_BUILD_WITH_BUNDLED_FREETDS=1;python -m  pip install pymssql
+  - python -m  pip install mysql-python
+install: # now just our code
+  - pip install git+https://github.com/ODM2/geoalchemy.git@odm2#egg=geoalchemy-0.7.3
+  - pip install   .
+  - pip install  -r requirements_tests.txt  --allow-external pyodbc --allow-unverified pyodbc
+    # pysqlite
+  - pip install pysqlite
+  - pip list
+
+# don't forget to open up the azure mssql server to these addreses
+# https://docs.travis-ci.com/user/ip-addresses/
+
+# command to run tests
+script:
+
+  - py.test
+

appveyor.yml

@@ -0,0 +1,171 @@
+environment:
+# patterned after: https://github.com/ogrisel/python-appveyor-demo/blob/master/appveyor.yml
+  global:
+    # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the
+    # /E:ON and /V:ON options are not enabled in the batch script intepreter
+    # See: http://stackoverflow.com/a/13751649/163740
+    CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\appveyor\\run_with_env.cmd"
+    # postgres
+    POSTGRES_PORT: tcp://localhost:5432
+    POSTGRES_ENV_POSTGRES_USER: postgres
+    POSTGRES_ENV_POSTGRES_PASSWORD: Password12!
+    POSTGRES_ENV_POSTGRES_DB: odm2
+    POSTGRES_PATH: C:\Program Files\PostgreSQL\9.4
+    PGUSER: postgres
+    PGPASSWORD: Password12!
+    # mysql
+    MYSQL_PORT: tcp://localhost:3306
+    MYSQL_ENV_MYSQL_USER: root
+    MYSQL_ENV_MYSQL_PASSWORD: Password12!
+    MYSQL_ENV_MYSQL_DATABASE: odm2
+    MYSQL_PATH: C:\Program Files\MySql\MySQL Server 5.6
+    MYSQL_PWD: Password12!
+    # sql server
+    SQLSERVER_ENV_SQLSERVER_HOST: localhost
+    SQLSERVER_ENV_SQLSERVER_PORT: 1433
+    SQLSERVER_ENV_SQLSERVER_USER: sa
+    SQLSERVER_ENV_SQLSERVER_PASSWORD: Password12!
+    SQLSERVER_ENV_SQLSERVER_DATABASE: odm2
+  matrix:
+
+
+    # Pre-installed Python versions, which Appveyor may upgrade to
+    # a later point release.
+    # See: http://www.appveyor.com/docs/installed-software#python
+
+
+    matrix:
+      - PYTHON: "C:\\Python27-conda32"
+        PYTHON_VERSION: "2.7"
+        PYTHON_ARCH: "32"
+
+      - PYTHON: "C:\\Python34-conda64"
+        PYTHON_VERSION: "3.4"
+        PYTHON_ARCH: "64"
+
+services:
+  - mssql2008r2sp2
+  - mysql
+  - postgresql
+
+install:
+  # If there is a newer build queued for the same PR, cancel this one.
+  # The AppVeyor 'rollout builds' option is supposed to serve the same
+  # purpose but it is problematic because it tends to cancel builds pushed
+  # directly to master instead of just PR builds (or the converse).
+  # credits: JuliaLang developers.
+  - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod `
+        https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | `
+        Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { `
+          throw "There are newer queued builds for this pull request, failing early." }
+  - ECHO "Filesystem root:"
+  - ps: "ls \"C:/\""
+
+  - ECHO "Installed SDKs:"
+  - ps: "ls \"C:/Program Files/Microsoft SDKs/Windows\""
+
+  # Install Python (from the official .msi of http://python.org) and pip when
+  # not already installed.
+  - ps: if (-not(Test-Path($env:PYTHON))) { & appveyor\install.ps1 }
+
+  # Prepend newly installed Python to the PATH of this build (this cannot be
+  # done from inside the powershell script as it would require to restart
+  # the parent CMD process).
+  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
+  # add databases
+  - "SET PATH=%POSTGRES_PATH%\bin;%MYSQL_PATH%\bin;%PATH%"
+
+  # Check that we have the expected version and architecture for Python
+  - "python --version"
+  - "python -c \"import struct; print(struct.calcsize('P') * 8)\""
+
+  # Upgrade to the latest version of pip to avoid it displaying warnings
+  # about it being out of date.
+  - "pip install --disable-pip-version-check --user --upgrade pip"
+
+  # Install the build dependencies of the project. If some dependencies contain
+  # compiled extensions and are not provided as pre-built wheel packages,
+  # pip will build them from source using the MSVC compiler matching the
+  # target Python version and architecture
+  - "%CMD_IN_ENV% pip install -r dev-requirements.txt"
+
+build_script:
+  # Build the compiled extension
+  - "%CMD_IN_ENV% python setup.py build"
+build_script:
+  # Build the compiled extension
+  - "%CMD_IN_ENV% python setup.py build"
+  build_script:
+  # postgres
+  - createdb odm2
+  - psql -d odm2 -a -f spec/databases/postgresql/schema/schema.sql
+  # mysql
+  - mysql -e "drop database test; create database odm2;" --user=root
+  - mysql sqlectron < spec/databases/mysql/schema/schema.sql --user=root
+  # sqlserver
+  - ps: ./appveyor-sqlserver.ps1 SQL2008R2SP2
+  - sqlcmd -S localhost,1433 -U sa -P Password12! -Q "CREATE DATABASE odm2" -d "master"
+  - sqlcmd -S localhost,1433 -U sa -P Password12! -i spec/databases/sqlserver/schema/schema.sql -d "odm2"
+  - sqlcmd -S localhost,1433 -U sa -P Password12! -Q "select table_name from information_schema.tables" -d "odm2"
+
+test_script:
+  # Run the project tests
+  - "%CMD_IN_ENV% python setup.py nosetests"
+
+language: python
+python:
+#  - "2.6"
+  - "2.7"
+#  - "3.2"
+#  - "3.3"
+#  - "3.4"
+#  - "3.5"
+#  - "3.5-dev" # 3.5 development branch
+#  - "nightly" # currently points to 3.6-dev
+# command to install dependencies
+
+before_install:
+#https://github.com/sqlectron/sqlectron-core/blob/master/appveyor.yml
+  - mysql -e "CREATE USER 'ODM'@'localhost' IDENTIFIED BY 'odm';GRANT ALL PRIVILEGES ON *.* TO 'ODM'@'localhost';" -uroot
+  - mysql -e "create database IF NOT EXISTS odm2;" -uroot
+  - mysql -e "create database IF NOT EXISTS odm2test;" -uroot
+  - psql -U postgres -c "create extension postgis"
+  - psql -c 'DROP DATABASE IF EXISTS odm2test;' -U postgres;
+  - psql -c 'create database odm2test;' -U postgres;
+#  - psql -U postgres -d odm2test -a -f ./tests/schemas/postgresql/ODM2_for_PostgreSQL.sql
+  - psql -c 'DROP DATABASE IF EXISTS odm2;' -U postgres;
+  - psql -c 'create database odm2;' -U postgres;
+  # patterned after: https://github.com/ptrv/gpx2spatialite/blob/master/.travis.yml
+#  - sudo apt-get install -y python-software-properties
+#  - sudo apt-add-repository -y ppa:git-core/ppa
+#  - sudo apt-add-repository -y ppa:ubuntugis/ppa
+#  - sudo apt-get update -qq
+#  - sudo apt-get install unixodbc  unixodbc-dev tdsodbc
+#  - sudo apt-get install freetds-dev freetds-bin
+#  - sudo apt-get install libc6  e2fsprogs  # mssql driver
+  # Spatialiate
+#  - sudo apt-get install -y libproj-dev libgeos-dev libspatialite-dev
+#  - sudo ln -s /usr/lib/x86_64-linux-gnu/libspatialite.so /usr/lib/libspatialite.so
+#  - sudo apt-get install python-scipy python-matplotlib python-pandas python-sympy python-nose
+#  - sudo apt-get install python-matplotlib python-pandas python-nose
+  - pip install pandas
+install: # now just our code
+  - pip install git+https://github.com/ODM2/geoalchemy.git@odm2#egg=geoalchemy-0.7.3
+  - pip install .
+  - pip install -r requirements_tests.txt  --allow-external pyodbc --allow-unverified pyodbc
+    # pysqlite
+  - pip install pysqlite
+  - dir   .\tests\usecasesql\littlebearriver\sampledatabases\odm2_mysql\LBR_MySQL_SmallExample.sql .\tests\usecasesql\marchantariats\marchantariats.sql
+  - mysql --user root --verbose  odm2 < .\tests\usecasesql\littlebearriver\sampledatabases\odm2_mysql\LBR_MySQL_SmallExample.sql
+# add -a to psql to see full log
+  - psql -U postgres   -f ./tests/usecasesql/marchantariats/marchantariats.sql
+
+# don't forget to open up the azure mssql server to these addreses
+# https://docs.travis-ci.com/user/ip-addresses/
+
+# command to run tests
+script:
+# just the connection part
+  - py.test tests/test_connection.py
+  - py.test
+

distribution_notes.txt

@@ -0,0 +1,32 @@
+
+Testing a package
+* upload to pyptest: https://wiki.python.org/moin/TestPyPI
+https://testpypi.python.org/pypi
+
+python setup.py sdist upload -r https://testpypi.python.org/pypi
+
+=== testing
+cd dev_odm
+virtualenv --system-site-packages v_odmtest
+cd ODM2PythonAPI
+python setup.py install
+# above failed on windows... need to be able to compile code
+
+=== CONDA == WINDOWS
+Start anaconda console
+conda search "^python$"
+conda create -n v_odm2test python=2.7.11 anaconda panadas sqlalchemy six geos pyodbc
+activate v_odm2test
+conda install -c SciTools shapely # to get binary versions
+
+
+conda remove --name v_odm2test --all
+# note issue solved by editing (anacona)\Scripts\activate.bat
+http://stackoverflow.com/questions/34818282/anaconda-prompt-loading-error-the-input-line-is-too-long
+===
+note for packaging yodatools
+https://gehrcke.de/2014/02/distributing-a-python-command-line-application/
+
+creating the databases:
+connection string for spatiallite
+sqlite+pysqlite:///../../../ODM2PythonAPI/tests/spatialite/odm2_test.sqlite

doc/DatabaseNotes.md

@@ -0,0 +1,15 @@
+Database Notes
+==============
+
+Supported Databases:
+* sqlite/Spatialite
+* Postgres
+* MySql
+* MsSql
+
+Generic Instructions:
+=====================
+# create database from Schema in ODM2 https://github.com/ODM2/ODM2/tree/master/src
+# determine appropriate connection string
+# populate database using the load_cv.py toolkit
+# Use YODA Tools.

doc/MySql.md

@@ -0,0 +1,47 @@
+MySql Notes:
+=========
+On Linux, you are required to use { lower_case_table_names=1 }. It is suggested that databases should use this setting.
+
+Connection String
+=================
+
+Create DB
+=========
+
+Populate with CV
+===============
+
+
+Linux Troubleshooting Notes
+===========================
+On Linux, you are required to use { lower_case_table_names=1 }.
+to see if this is set type:
+mysql --verbose -e "show variables like 'lower%';" --user=root
+--------------
+show variables like 'lower%'
+--------------
++------------------------+-------+
+| Variable_name          | Value |
++------------------------+-------+
+| lower_case_file_system | OFF   |
+| lower_case_table_names | 1     |
++------------------------+-------+
+
+If not you can install this line in $HOME/.my.cnf, /etc/my.cnf, or /etc/mysql/my.cnf
+
+[mysqld]
+lower-case-table-names = 1
+
+to see what files are being used by mysql:
+> strace mysql 2>&1 | grep cnf
+
+stat("/etc/my.cnf", 0x7ffef7c8dca0)     = -1 ENOENT (No such file or directory)
+stat("/etc/mysql/my.cnf", {st_mode=S_IFREG|0644, st_size=5232, ...}) = 0
+open("/etc/mysql/my.cnf", O_RDONLY|O_CLOEXEC) = 3
+stat("/etc/mysql/conf.d/mariadb.cnf", {st_mode=S_IFREG|0644, st_size=435, ...}) = 0
+open("/etc/mysql/conf.d/mariadb.cnf", O_RDONLY|O_CLOEXEC) = 4
+stat("/etc/mysql/conf.d/mysqld_safe_syslog.cnf", {st_mode=S_IFREG|0644, st_size=36, ...}) = 0
+open("/etc/mysql/conf.d/mysqld_safe_syslog.cnf", O_RDONLY|O_CLOEXEC) = 4
+stat("/etc/mysql/conf.d/tokudb.cnf", {st_mode=S_IFREG|0644, st_size=285, ...}) = 0
+open("/etc/mysql/conf.d/tokudb.cnf", O_RDONLY|O_CLOEXEC) = 4
+stat("/home/travis/.my.cnf", 0x7ffef7c8dca0) = -1 ENOENT (No such file or directory)

notes_sqlacodegen.txt

@@ -0,0 +1,8 @@
+
+https://pypi.python.org/pypi/sqlacodegen
+
+pip install sqlacodegen
+
+sqlacodegen  "mssql+pyodbc://user:password@localhost/ODM2?driver=SQL+Server+Native+Client+10.0"
+
+sqlacodegen  "mssql+pyodbc://user:password@localhost/ODM2_201512?driver=SQL+Server+Native+Client+10.0" --schema ODM2 --outfile generated_schema_201512.py