From 2f754207690ab40f29ba1d912c29373438a10852 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Thu, 15 Aug 2024 16:48:00 -0700 Subject: [PATCH 01/20] Add `pytest` as an extra requirement --- setup.py | 154 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 78 insertions(+), 76 deletions(-) diff --git a/setup.py b/setup.py index ddcc626d..b416b9fd 100644 --- a/setup.py +++ b/setup.py @@ -19,98 +19,100 @@ def select_constraint(default, nightly=None, git_master=None): - """Select dependency constraint based on TFX_DEPENDENCY_SELECTOR env var.""" - selector = os.environ.get('TFX_DEPENDENCY_SELECTOR') - if selector == 'UNCONSTRAINED': - return '' - elif selector == 'NIGHTLY' and nightly is not None: - return nightly - elif selector == 'GIT_MASTER' and git_master is not None: - return git_master - else: - return default + """Select dependency constraint based on TFX_DEPENDENCY_SELECTOR env var.""" + selector = os.environ.get("TFX_DEPENDENCY_SELECTOR") + if selector == "UNCONSTRAINED": + return "" + elif selector == "NIGHTLY" and nightly is not None: + return nightly + elif selector == "GIT_MASTER" and git_master is not None: + return git_master + else: + return default # Get version from version module. -with open('tensorflow_transform/version.py') as fp: - globals_dict = {} - exec(fp.read(), globals_dict) # pylint: disable=exec-used -__version__ = globals_dict['__version__'] +with open("tensorflow_transform/version.py") as fp: + globals_dict = {} + exec(fp.read(), globals_dict) # pylint: disable=exec-used +__version__ = globals_dict["__version__"] def _make_required_install_packages(): - # Make sure to sync the versions of common dependencies (absl-py, numpy, and - # protobuf) with TF and pyarrow version with tfx-bsl. - return [ - 'absl-py>=0.9,<2.0.0', - 'apache-beam[gcp]>=2.53,<3;python_version>="3.11"', - 'apache-beam[gcp]>=2.47,<3;python_version<"3.11"', - 'numpy>=1.22.0', - 'protobuf>=4.25.2,<5;python_version>="3.11"', - 'protobuf>=3.20.3,<5;python_version<"3.11"', - 'pyarrow>=10,<11', - 'pydot>=1.2,<2', - 'tensorflow' - + select_constraint( - default='>=2.15,<2.16', - nightly='>=2.16.0.dev', - git_master='@git+https://github.com/tensorflow/tensorflow@master', - ), - 'tensorflow-metadata' - + select_constraint( - default='>=1.15.0,<1.16.0', - nightly='>=1.16.0.dev', - git_master='@git+https://github.com/tensorflow/metadata@master', - ), - 'tfx-bsl' - + select_constraint( - default='>=1.15.1,<1.16.0', - nightly='>=1.16.0.dev', - git_master='@git+https://github.com/tensorflow/tfx-bsl@master', - ), - ] + # Make sure to sync the versions of common dependencies (absl-py, numpy, and + # protobuf) with TF and pyarrow version with tfx-bsl. + return [ + "absl-py>=0.9,<2.0.0", + 'apache-beam[gcp]>=2.53,<3;python_version>="3.11"', + 'apache-beam[gcp]>=2.47,<3;python_version<"3.11"', + "numpy>=1.22.0", + 'protobuf>=4.25.2,<5;python_version>="3.11"', + 'protobuf>=3.20.3,<5;python_version<"3.11"', + "pyarrow>=10,<11", + "pydot>=1.2,<2", + "tensorflow" + + select_constraint( + default=">=2.15,<2.16", + nightly=">=2.16.0.dev", + git_master="@git+https://github.com/tensorflow/tensorflow@master", + ), + "tensorflow-metadata" + + select_constraint( + default=">=1.15.0,<1.16.0", + nightly=">=1.16.0.dev", + git_master="@git+https://github.com/tensorflow/metadata@master", + ), + "tfx-bsl" + + select_constraint( + default=">=1.15.1,<1.16.0", + nightly=">=1.16.0.dev", + git_master="@git+https://github.com/tensorflow/tfx-bsl@master", + ), + ] # Get the long description from the README file. -with open('README.md') as fp: - _LONG_DESCRIPTION = fp.read() +with open("README.md") as fp: + _LONG_DESCRIPTION = fp.read() setup( - name='tensorflow-transform', + name="tensorflow-transform", version=__version__, - author='Google Inc.', - author_email='tensorflow-extended-dev@googlegroups.com', - license='Apache 2.0', + author="Google Inc.", + author_email="tensorflow-extended-dev@googlegroups.com", + license="Apache 2.0", classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Intended Audience :: Education', - 'Intended Audience :: Science/Research', - 'License :: OSI Approved :: Apache Software License', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3 :: Only', - 'Topic :: Scientific/Engineering', - 'Topic :: Scientific/Engineering :: Artificial Intelligence', - 'Topic :: Scientific/Engineering :: Mathematics', - 'Topic :: Software Development', - 'Topic :: Software Development :: Libraries', - 'Topic :: Software Development :: Libraries :: Python Modules', + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Education", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3 :: Only", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Scientific/Engineering :: Mathematics", + "Topic :: Software Development", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules", ], namespace_packages=[], install_requires=_make_required_install_packages(), - python_requires='>=3.9,<4', + extras_require={"test": ["pytest>=8.0"]}, + python_requires=">=3.9,<4", packages=find_packages(), include_package_data=True, - package_data={'tensorflow_transform': ['py.typed']}, - description='A library for data preprocessing with TensorFlow', + package_data={"tensorflow_transform": ["py.typed"]}, + description="A library for data preprocessing with TensorFlow", long_description=_LONG_DESCRIPTION, - long_description_content_type='text/markdown', - keywords='tensorflow transform tfx', - url='https://www.tensorflow.org/tfx/transform/get_started', - download_url='https://github.com/tensorflow/transform/tags', - requires=[]) + long_description_content_type="text/markdown", + keywords="tensorflow transform tfx", + url="https://www.tensorflow.org/tfx/transform/get_started", + download_url="https://github.com/tensorflow/transform/tags", + requires=[], +) From 386d0898d5e422af273a032acf6c6561ce6e28ef Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Thu, 15 Aug 2024 16:48:15 -0700 Subject: [PATCH 02/20] Add `pytest.ini` --- pytest.ini | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..82490cd4 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,9 @@ +[pytest] +addopts = --verbose --import-mode=importlib +testpaths = tensorflow_transform +python_files = *_test.py +norecursedirs = .* *.egg +log_format = %(asctime)s %(levelname)s %(message)s +log_date_format = %Y-%m-%d %H:%M:%S +log_cli = True +log_cli_level = INFO \ No newline at end of file From c9d54fc7b5a16c84374488f4f43ca2934547437a Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Thu, 15 Aug 2024 16:54:26 -0700 Subject: [PATCH 03/20] Add first pass at a GitHub workflow that runs tests --- .github/workflows/ci-test.yml | 39 +++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/ci-test.yml diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml new file mode 100644 index 00000000..b6f5c230 --- /dev/null +++ b/.github/workflows/ci-test.yml @@ -0,0 +1,39 @@ +# Github action definitions for unit-tests with PRs. + +name: tft-unit-tests +on: + pull_request: + branches: [ master ] + paths-ignore: + - '**.md' + - 'docs/**' + workflow_dispatch: + push: # DEBUG: Remove before merge + +jobs: + unit-tests: + if: github.actor != 'copybara-service[bot]' + runs-on: ubuntu-latest + timeout-minutes: 60 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: '3.9' + cache: 'pip' + cache-dependency-path: | + setup.py + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade wheel setuptools + python -m pip install -e .[test] + + - name: Run unit tests + shell: bash + run: | + pytest From 646ec4b0dbc8e79f873bdf1faca77c92be684369 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Thu, 15 Aug 2024 17:04:48 -0700 Subject: [PATCH 04/20] Add matrix for python versions --- .github/workflows/ci-test.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index b6f5c230..76c72784 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -16,13 +16,18 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 + version_matrix: + strategy: + matrix: + python-version: ['3.9', '3.10', '3.11'] + steps: - uses: actions/checkout@v4 - - name: Set up Python 3.9 + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: ${{ matrix.python-version }} cache: 'pip' cache-dependency-path: | setup.py From 230e73ab71025fdfb72ad8bca3aa0329e97d517a Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Thu, 15 Aug 2024 17:06:45 -0700 Subject: [PATCH 05/20] Fix typo --- .github/workflows/ci-test.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 76c72784..b75d4908 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -16,10 +16,9 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 - version_matrix: - strategy: - matrix: - python-version: ['3.9', '3.10', '3.11'] + strategy: + matrix: + python-version: ['3.9', '3.10', '3.11'] steps: - uses: actions/checkout@v4 From 13e62cb4a99f91bd97130095c7775d118a768ebe Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Thu, 15 Aug 2024 17:30:26 -0700 Subject: [PATCH 06/20] Remove debugging trigger --- .github/workflows/ci-test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index b75d4908..4f98051e 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -8,7 +8,6 @@ on: - '**.md' - 'docs/**' workflow_dispatch: - push: # DEBUG: Remove before merge jobs: unit-tests: From 7c287e7c8df1fc65987c650ffe19e2476091e28f Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Thu, 15 Aug 2024 19:38:51 -0700 Subject: [PATCH 07/20] Don't use editable install. Instead use regular install --- .github/workflows/ci-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 4f98051e..b76ee1f6 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -34,7 +34,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install --upgrade wheel setuptools - python -m pip install -e .[test] + python -m pip install .[test] - name: Run unit tests shell: bash From 764366559f99601d48838e9c40042f2323a7a8a7 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Thu, 15 Aug 2024 20:14:13 -0700 Subject: [PATCH 08/20] Update install instructions --- README.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 87808e54..12db4e9e 100644 --- a/README.md +++ b/README.md @@ -48,17 +48,23 @@ source /bin/activate pip3 install setuptools wheel git clone https://github.com/tensorflow/transform.git cd transform -python3 setup.py bdist_wheel +pip3 install . ``` -This will build the TFT wheel in the dist directory. To install the wheel from -dist directory run the commands +If you are doing development on the TFT repo, replace ``` -cd dist -pip3 install tensorflow_transform--py3-none-any.whl +pip3 install . ``` +with + +``` +pip3 install -e . +``` + +The `-e` flag causes TFT to be installed in [development mode](https://setuptools.pypa.io/en/latest/userguide/development_mode.html). + ### Nightly Packages TFT also hosts nightly packages at https://pypi-nightly.tensorflow.org on From f305a18d2fa25e29f39426b8e63240ddcafaa849 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Thu, 15 Aug 2024 20:53:04 -0700 Subject: [PATCH 09/20] Add note about running tests --- README.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 12db4e9e..4ea28b7d 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ pip install tensorflow-transform To build from source follow the following steps: Create a virtual environment by running the commands -``` +```bash python3 -m venv source /bin/activate pip3 install setuptools wheel @@ -53,13 +53,13 @@ pip3 install . If you are doing development on the TFT repo, replace -``` +```bash pip3 install . ``` with -``` +```bash pip3 install -e . ``` @@ -78,6 +78,14 @@ pip install --extra-index-url https://pypi-nightly.tensorflow.org/simple tensorf This will install the nightly packages for the major dependencies of TFT such as TensorFlow Metadata (TFMD), TFX Basic Shared Libraries (TFX-BSL). +### Running Tests + +To run TFT tests, run the following command from the root of the repository: + +```bash +pytest +``` + ### Notable Dependencies TensorFlow is required. From ab5f4f88f462eb9b0f84599461ef4be502c9d773 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Thu, 15 Aug 2024 20:57:41 -0700 Subject: [PATCH 10/20] Remove `if __name__ == "__main__":` from test files --- tensorflow_transform/analyzers_test.py | 2 -- tensorflow_transform/annotators_test.py | 2 -- tensorflow_transform/beam/analysis_graph_builder_test.py | 2 -- tensorflow_transform/beam/analyzer_cache_test.py | 2 -- tensorflow_transform/beam/analyzer_impls_test.py | 2 -- tensorflow_transform/beam/annotators_test.py | 2 -- tensorflow_transform/beam/bucketize_integration_test.py | 2 -- tensorflow_transform/beam/cached_impl_test.py | 2 -- tensorflow_transform/beam/combiner_packing_util_test.py | 2 -- tensorflow_transform/beam/context_test.py | 2 -- tensorflow_transform/beam/deep_copy_test.py | 2 -- tensorflow_transform/beam/impl_output_record_batches_test.py | 2 -- tensorflow_transform/beam/impl_test.py | 2 -- tensorflow_transform/beam/tft_beam_io/beam_metadata_io_test.py | 2 -- tensorflow_transform/beam/tft_beam_io/transform_fn_io_test.py | 2 -- tensorflow_transform/beam/tukey_hh_params_integration_test.py | 2 -- tensorflow_transform/beam/vocabulary_integration_test.py | 2 -- .../beam/vocabulary_tfrecord_gzip_integration_test.py | 2 -- tensorflow_transform/coders/csv_coder_test.py | 2 -- tensorflow_transform/coders/example_proto_coder_test.py | 2 -- tensorflow_transform/common_test.py | 2 -- tensorflow_transform/gaussianization_test.py | 2 -- tensorflow_transform/graph_tools_test.py | 2 -- tensorflow_transform/impl_helper_test.py | 2 -- tensorflow_transform/info_theory_test.py | 2 -- tensorflow_transform/inspect_preprocessing_fn_test.py | 2 -- tensorflow_transform/mappers_test.py | 2 -- tensorflow_transform/nodes_test.py | 2 -- tensorflow_transform/pretrained_models_test.py | 2 -- tensorflow_transform/saved/saved_model_loader_test.py | 2 -- tensorflow_transform/saved/saved_transform_io_test.py | 2 -- tensorflow_transform/saved/saved_transform_io_v2_test.py | 2 -- tensorflow_transform/schema_inference_test.py | 2 -- tensorflow_transform/test_case_test.py | 2 -- tensorflow_transform/tf2_utils_test.py | 2 -- tensorflow_transform/tf_metadata/dataset_metadata_test.py | 2 -- tensorflow_transform/tf_metadata/metadata_io_test.py | 2 -- tensorflow_transform/tf_metadata/schema_utils_test.py | 2 -- tensorflow_transform/tf_utils_test.py | 2 -- 39 files changed, 78 deletions(-) diff --git a/tensorflow_transform/analyzers_test.py b/tensorflow_transform/analyzers_test.py index 8190b496..7b6c2ac1 100644 --- a/tensorflow_transform/analyzers_test.py +++ b/tensorflow_transform/analyzers_test.py @@ -624,5 +624,3 @@ def testMinDiffFromAvg(self): analyzers.calculate_recommended_min_diff_from_avg(100000000), 25) -if __name__ == '__main__': - test_case.main() diff --git a/tensorflow_transform/annotators_test.py b/tensorflow_transform/annotators_test.py index 70583319..0c624890 100644 --- a/tensorflow_transform/annotators_test.py +++ b/tensorflow_transform/annotators_test.py @@ -65,5 +65,3 @@ def preprocessing_fn(): self.assertEqual(trackable_object, object_tracker.trackable_objects[0]) -if __name__ == '__main__': - test_case.main() diff --git a/tensorflow_transform/beam/analysis_graph_builder_test.py b/tensorflow_transform/beam/analysis_graph_builder_test.py index b1aca33c..f9252e1d 100644 --- a/tensorflow_transform/beam/analysis_graph_builder_test.py +++ b/tensorflow_transform/beam/analysis_graph_builder_test.py @@ -579,5 +579,3 @@ class _Analyzer( structured_outputs) -if __name__ == '__main__': - tft_unit.main() diff --git a/tensorflow_transform/beam/analyzer_cache_test.py b/tensorflow_transform/beam/analyzer_cache_test.py index b993477d..fdf08309 100644 --- a/tensorflow_transform/beam/analyzer_cache_test.py +++ b/tensorflow_transform/beam/analyzer_cache_test.py @@ -310,5 +310,3 @@ def expand(self, pbegin): beam_test_util.equal_to([test_cache_dict[key].cache_dict['b']])) -if __name__ == '__main__': - test_case.main() diff --git a/tensorflow_transform/beam/analyzer_impls_test.py b/tensorflow_transform/beam/analyzer_impls_test.py index 671977cd..af66115e 100644 --- a/tensorflow_transform/beam/analyzer_impls_test.py +++ b/tensorflow_transform/beam/analyzer_impls_test.py @@ -136,5 +136,3 @@ def testJoinBoundarieRows(self, input_boundaries, expected_boundaries, self.assertAllEqual(num_buckets, expected_num_buckets) -if __name__ == '__main__': - tft_unit.main() diff --git a/tensorflow_transform/beam/annotators_test.py b/tensorflow_transform/beam/annotators_test.py index 5b016d37..77890ef4 100644 --- a/tensorflow_transform/beam/annotators_test.py +++ b/tensorflow_transform/beam/annotators_test.py @@ -259,5 +259,3 @@ def preprocessing_fn(inputs): ) -if __name__ == '__main__': - tft_unit.main() diff --git a/tensorflow_transform/beam/bucketize_integration_test.py b/tensorflow_transform/beam/bucketize_integration_test.py index 3d231eb2..7ce9a62d 100644 --- a/tensorflow_transform/beam/bucketize_integration_test.py +++ b/tensorflow_transform/beam/bucketize_integration_test.py @@ -888,5 +888,3 @@ def testBucketizationSpecificDistribution(self): inputs, expected_boundaries, tf.float32, num_buckets=5) -if __name__ == '__main__': - tft_unit.main() diff --git a/tensorflow_transform/beam/cached_impl_test.py b/tensorflow_transform/beam/cached_impl_test.py index fdaf1da3..e5f89006 100644 --- a/tensorflow_transform/beam/cached_impl_test.py +++ b/tensorflow_transform/beam/cached_impl_test.py @@ -1749,5 +1749,3 @@ def preprocessing_fn(inputs): ) -if __name__ == '__main__': - tft_unit.main() diff --git a/tensorflow_transform/beam/combiner_packing_util_test.py b/tensorflow_transform/beam/combiner_packing_util_test.py index b1ac44a5..4ba643de 100644 --- a/tensorflow_transform/beam/combiner_packing_util_test.py +++ b/tensorflow_transform/beam/combiner_packing_util_test.py @@ -433,5 +433,3 @@ def _side_effect_fn(saved_model_future, cache_value_nodes, second=expected_dot_graph_str_after_packing) -if __name__ == '__main__': - test_case.main() diff --git a/tensorflow_transform/beam/context_test.py b/tensorflow_transform/beam/context_test.py index 8115af9c..cd3cc3ca 100644 --- a/tensorflow_transform/beam/context_test.py +++ b/tensorflow_transform/beam/context_test.py @@ -40,5 +40,3 @@ def testNestedContextCreateBaseTempDir(self): tft_beam.Context.create_base_temp_dir() -if __name__ == '__main__': - tft_unit.main() diff --git a/tensorflow_transform/beam/deep_copy_test.py b/tensorflow_transform/beam/deep_copy_test.py index 374463de..ae62137d 100644 --- a/tensorflow_transform/beam/deep_copy_test.py +++ b/tensorflow_transform/beam/deep_copy_test.py @@ -330,5 +330,3 @@ def testDeepCopyTags(self): self.assertEqual(DeepCopyTest._counts['Add2'], 3 * (num_copies + 1)) self.assertEqual(DeepCopyTest._counts['Add3'], 3) -if __name__ == '__main__': - unittest.main() diff --git a/tensorflow_transform/beam/impl_output_record_batches_test.py b/tensorflow_transform/beam/impl_output_record_batches_test.py index a4c01bde..a2bb2b5f 100644 --- a/tensorflow_transform/beam/impl_output_record_batches_test.py +++ b/tensorflow_transform/beam/impl_output_record_batches_test.py @@ -199,5 +199,3 @@ def testConvertToLargeRecordBatch( self.assertGreater(actual_num_batches, 1) -if __name__ == '__main__': - tft_unit.main() diff --git a/tensorflow_transform/beam/impl_test.py b/tensorflow_transform/beam/impl_test.py index 77e87417..4cbb9fb0 100644 --- a/tensorflow_transform/beam/impl_test.py +++ b/tensorflow_transform/beam/impl_test.py @@ -4797,5 +4797,3 @@ def test_preprocessing_fn_returns_wrong_type(self): expected_data=None) -if __name__ == '__main__': - tft_unit.main() diff --git a/tensorflow_transform/beam/tft_beam_io/beam_metadata_io_test.py b/tensorflow_transform/beam/tft_beam_io/beam_metadata_io_test.py index 0dad77e1..9c51e611 100644 --- a/tensorflow_transform/beam/tft_beam_io/beam_metadata_io_test.py +++ b/tensorflow_transform/beam/tft_beam_io/beam_metadata_io_test.py @@ -92,5 +92,3 @@ def mock_write_metadata(metadata, path): self.assertEqual(metadata, test_metadata.COMPLETE_METADATA) -if __name__ == '__main__': - tf.test.main() diff --git a/tensorflow_transform/beam/tft_beam_io/transform_fn_io_test.py b/tensorflow_transform/beam/tft_beam_io/transform_fn_io_test.py index c40c85e4..5fc146e9 100644 --- a/tensorflow_transform/beam/tft_beam_io/transform_fn_io_test.py +++ b/tensorflow_transform/beam/tft_beam_io/transform_fn_io_test.py @@ -149,5 +149,3 @@ def mock_copy_tree_to_unique_temp_dir(source, base_temp_dir_path): self.assertEqual(2, len(file_io.list_directory(transform_output_dir))) -if __name__ == '__main__': - tf.test.main() diff --git a/tensorflow_transform/beam/tukey_hh_params_integration_test.py b/tensorflow_transform/beam/tukey_hh_params_integration_test.py index 3f4dee0f..05d28a01 100644 --- a/tensorflow_transform/beam/tukey_hh_params_integration_test.py +++ b/tensorflow_transform/beam/tukey_hh_params_integration_test.py @@ -627,5 +627,3 @@ def assert_and_cast_dtype(tensor): # Runs the test deterministically on the whole batch. beam_pipeline=beam.Pipeline()) -if __name__ == '__main__': - tft_unit.main() diff --git a/tensorflow_transform/beam/vocabulary_integration_test.py b/tensorflow_transform/beam/vocabulary_integration_test.py index 437d794d..2899d91a 100644 --- a/tensorflow_transform/beam/vocabulary_integration_test.py +++ b/tensorflow_transform/beam/vocabulary_integration_test.py @@ -2088,5 +2088,3 @@ def preprocessing_fn(inputs): ) -if __name__ == '__main__': - tft_unit.main() diff --git a/tensorflow_transform/beam/vocabulary_tfrecord_gzip_integration_test.py b/tensorflow_transform/beam/vocabulary_tfrecord_gzip_integration_test.py index 4ac011c2..9b49eaa0 100644 --- a/tensorflow_transform/beam/vocabulary_tfrecord_gzip_integration_test.py +++ b/tensorflow_transform/beam/vocabulary_tfrecord_gzip_integration_test.py @@ -25,5 +25,3 @@ def _VocabFormat(self): return 'tfrecord_gzip' -if __name__ == '__main__': - tft_unit.main() diff --git a/tensorflow_transform/coders/csv_coder_test.py b/tensorflow_transform/coders/csv_coder_test.py index a7ddcfa2..a6fdc49a 100644 --- a/tensorflow_transform/coders/csv_coder_test.py +++ b/tensorflow_transform/coders/csv_coder_test.py @@ -292,5 +292,3 @@ def test_picklable(self): self.assertEqual(coder.encode(instance), csv_line.encode('utf-8')) -if __name__ == '__main__': - test_case.main() diff --git a/tensorflow_transform/coders/example_proto_coder_test.py b/tensorflow_transform/coders/example_proto_coder_test.py index f8738160..7f384f78 100644 --- a/tensorflow_transform/coders/example_proto_coder_test.py +++ b/tensorflow_transform/coders/example_proto_coder_test.py @@ -406,5 +406,3 @@ def test_example_proto_coder_cache(self): self.assertSerializedProtosEqual(coder.encode(instance), serialized_proto) -if __name__ == '__main__': - test_case.main() diff --git a/tensorflow_transform/common_test.py b/tensorflow_transform/common_test.py index df9171a2..df4bf4e4 100644 --- a/tensorflow_transform/common_test.py +++ b/tensorflow_transform/common_test.py @@ -69,5 +69,3 @@ def fn2(): self.assertAllEqual([], graph.get_collection("another_collection")) -if __name__ == "__main__": - test_case.main() diff --git a/tensorflow_transform/gaussianization_test.py b/tensorflow_transform/gaussianization_test.py index e3891615..01262f54 100644 --- a/tensorflow_transform/gaussianization_test.py +++ b/tensorflow_transform/gaussianization_test.py @@ -252,5 +252,3 @@ def test_inverse_tukey_hh(self, samples, hl, hr, expected_output): self.assertAllClose(output, expected_output) -if __name__ == '__main__': - test_case.main() diff --git a/tensorflow_transform/graph_tools_test.py b/tensorflow_transform/graph_tools_test.py index c5caaa17..2b83459e 100644 --- a/tensorflow_transform/graph_tools_test.py +++ b/tensorflow_transform/graph_tools_test.py @@ -1264,5 +1264,3 @@ def _value_to_matcher(value, add_quotes=False): type(value), value)) -if __name__ == '__main__': - test_case.main() diff --git a/tensorflow_transform/impl_helper_test.py b/tensorflow_transform/impl_helper_test.py index ff3a0c90..e5a5601a 100644 --- a/tensorflow_transform/impl_helper_test.py +++ b/tensorflow_transform/impl_helper_test.py @@ -974,5 +974,3 @@ def iteration(counter, x_minus_counter): cond=stop_condition, body=iteration, loop_vars=initial_values)[1] -if __name__ == '__main__': - test_case.main() diff --git a/tensorflow_transform/info_theory_test.py b/tensorflow_transform/info_theory_test.py index b8136b4f..26dc87e0 100644 --- a/tensorflow_transform/info_theory_test.py +++ b/tensorflow_transform/info_theory_test.py @@ -166,5 +166,3 @@ def test_mutual_information(self, cell_count, row_count, col_count, self.assertNear(per_cell_mi, expected_mi, EPSILON) -if __name__ == '__main__': - unittest.main() diff --git a/tensorflow_transform/inspect_preprocessing_fn_test.py b/tensorflow_transform/inspect_preprocessing_fn_test.py index 7e37f61b..14947e54 100644 --- a/tensorflow_transform/inspect_preprocessing_fn_test.py +++ b/tensorflow_transform/inspect_preprocessing_fn_test.py @@ -166,5 +166,3 @@ def test_column_inference(self, preprocessing_fn, expected_transform_input_columns) -if __name__ == '__main__': - test_case.main() diff --git a/tensorflow_transform/mappers_test.py b/tensorflow_transform/mappers_test.py index ea3049b9..d92f7ceb 100644 --- a/tensorflow_transform/mappers_test.py +++ b/tensorflow_transform/mappers_test.py @@ -931,5 +931,3 @@ def testEstimatedProbabilityDensityMissingKey(self): self.assertAllEqual(expected, sess.run(result)) -if __name__ == '__main__': - test_case.main() diff --git a/tensorflow_transform/nodes_test.py b/tensorflow_transform/nodes_test.py index 9699a8f0..aca5151c 100644 --- a/tensorflow_transform/nodes_test.py +++ b/tensorflow_transform/nodes_test.py @@ -219,5 +219,3 @@ def testGetDotGraph(self): msg='Result dot graph is:\n{}'.format(dot_string)) -if __name__ == '__main__': - test_case.main() diff --git a/tensorflow_transform/pretrained_models_test.py b/tensorflow_transform/pretrained_models_test.py index 9d729f65..b67ae636 100644 --- a/tensorflow_transform/pretrained_models_test.py +++ b/tensorflow_transform/pretrained_models_test.py @@ -154,5 +154,3 @@ def testApplyFunctionWithCheckpointTwoInputs(self): self.assertAllEqual(output_value, [-1, 2, 5]) -if __name__ == '__main__': - tf.test.main() diff --git a/tensorflow_transform/saved/saved_model_loader_test.py b/tensorflow_transform/saved/saved_model_loader_test.py index 70b8f656..20735a06 100644 --- a/tensorflow_transform/saved/saved_model_loader_test.py +++ b/tensorflow_transform/saved/saved_model_loader_test.py @@ -46,5 +46,3 @@ def setUpClass(cls): # This class has no tests at the moment. -if __name__ == '__main__': - unittest.main() diff --git a/tensorflow_transform/saved/saved_transform_io_test.py b/tensorflow_transform/saved/saved_transform_io_test.py index 30ee65ad..d2aa06b6 100644 --- a/tensorflow_transform/saved/saved_transform_io_test.py +++ b/tensorflow_transform/saved/saved_transform_io_test.py @@ -305,5 +305,3 @@ def test_stale_asset_collections_are_cleaned(self): saved_transform_io.write_saved_transform_from_session( session, inputs, outputs, export_path) -if __name__ == '__main__': - tf.test.main() diff --git a/tensorflow_transform/saved/saved_transform_io_v2_test.py b/tensorflow_transform/saved/saved_transform_io_v2_test.py index c6d2a062..fd4bf51a 100644 --- a/tensorflow_transform/saved/saved_transform_io_v2_test.py +++ b/tensorflow_transform/saved/saved_transform_io_v2_test.py @@ -666,5 +666,3 @@ def func(inputs): self.assertEqual(restored_function(**input_kwargs)['a+b'], expected_output) -if __name__ == '__main__': - test_case.main() diff --git a/tensorflow_transform/schema_inference_test.py b/tensorflow_transform/schema_inference_test.py index a015983c..a67d8a66 100644 --- a/tensorflow_transform/schema_inference_test.py +++ b/tensorflow_transform/schema_inference_test.py @@ -402,5 +402,3 @@ def preprocessing_fn(_): self.assertProtoEquals(expected_schema, schema) -if __name__ == '__main__': - unittest.main() diff --git a/tensorflow_transform/test_case_test.py b/tensorflow_transform/test_case_test.py index 42914f6f..642c815a 100644 --- a/tensorflow_transform/test_case_test.py +++ b/tensorflow_transform/test_case_test.py @@ -80,5 +80,3 @@ def testSampleParametrizedTestMethod(self, my_arg, my_other_arg): self.assertIn((my_arg, my_other_arg), {(1, 'a'), (2, 'b')}) -if __name__ == '__main__': - unittest.main() diff --git a/tensorflow_transform/tf2_utils_test.py b/tensorflow_transform/tf2_utils_test.py index fafc5b81..299812f3 100644 --- a/tensorflow_transform/tf2_utils_test.py +++ b/tensorflow_transform/tf2_utils_test.py @@ -111,5 +111,3 @@ def foo(inputs): self.assertEqual(result['x_2'].dtype, dtype) -if __name__ == '__main__': - test_case.main() diff --git a/tensorflow_transform/tf_metadata/dataset_metadata_test.py b/tensorflow_transform/tf_metadata/dataset_metadata_test.py index e4576b1b..ca18a4ef 100644 --- a/tensorflow_transform/tf_metadata/dataset_metadata_test.py +++ b/tensorflow_transform/tf_metadata/dataset_metadata_test.py @@ -26,5 +26,3 @@ def test_sanity(self): self.assertEqual(metadata.schema, test_common.get_test_schema()) -if __name__ == '__main__': - unittest.main() diff --git a/tensorflow_transform/tf_metadata/metadata_io_test.py b/tensorflow_transform/tf_metadata/metadata_io_test.py index 097139ee..f6b420c5 100644 --- a/tensorflow_transform/tf_metadata/metadata_io_test.py +++ b/tensorflow_transform/tf_metadata/metadata_io_test.py @@ -134,5 +134,3 @@ def test_write_and_read(self): self.assertEqual(original, reloaded) -if __name__ == '__main__': - unittest.main() diff --git a/tensorflow_transform/tf_metadata/schema_utils_test.py b/tensorflow_transform/tf_metadata/schema_utils_test.py index 90d48145..36744460 100644 --- a/tensorflow_transform/tf_metadata/schema_utils_test.py +++ b/tensorflow_transform/tf_metadata/schema_utils_test.py @@ -80,5 +80,3 @@ def test_pop_ragged_source_columns(self, name, tensor_representation, self.assertEqual(feature_by_name, truncated_feature_by_name) -if __name__ == '__main__': - unittest.main() diff --git a/tensorflow_transform/tf_utils_test.py b/tensorflow_transform/tf_utils_test.py index 3edf5ddb..c296d918 100644 --- a/tensorflow_transform/tf_utils_test.py +++ b/tensorflow_transform/tf_utils_test.py @@ -2521,5 +2521,3 @@ def foo(input_tensor): self.assertAllEqual(output_tensor, expected_data) -if __name__ == '__main__': - test_case.main() From 843c39451522b3bbb8bb3dc2fcc44255ae6ec4b8 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Thu, 15 Aug 2024 22:28:41 -0700 Subject: [PATCH 11/20] Remove unnecessary packages from testing workflow --- .github/workflows/ci-test.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index b76ee1f6..e9ea140a 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -32,9 +32,7 @@ jobs: - name: Install dependencies run: | - python -m pip install --upgrade pip - python -m pip install --upgrade wheel setuptools - python -m pip install .[test] + pip install .[test] - name: Run unit tests shell: bash From fa0013c0a6bc3d7ad7654ebdba60c268f3fbc4dd Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Thu, 15 Aug 2024 22:29:24 -0700 Subject: [PATCH 12/20] Change `pip3` to `pip` --- README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4ea28b7d..08c1d44e 100644 --- a/README.md +++ b/README.md @@ -45,22 +45,21 @@ Create a virtual environment by running the commands ```bash python3 -m venv source /bin/activate -pip3 install setuptools wheel git clone https://github.com/tensorflow/transform.git cd transform -pip3 install . +pip install . ``` If you are doing development on the TFT repo, replace ```bash -pip3 install . +pip install . ``` with ```bash -pip3 install -e . +pip install -e . ``` The `-e` flag causes TFT to be installed in [development mode](https://setuptools.pypa.io/en/latest/userguide/development_mode.html). From 6b0f11ab5d9139229ed25e5e2dcffc5eafc3d339 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Thu, 15 Aug 2024 23:27:32 -0700 Subject: [PATCH 13/20] Change `python3` to `python` --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 08c1d44e..5693a427 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ To build from source follow the following steps: Create a virtual environment by running the commands ```bash -python3 -m venv +python -m venv source /bin/activate git clone https://github.com/tensorflow/transform.git cd transform From a0e64637ff80caac1e2278b4906ad7ef1234142d Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Thu, 15 Aug 2024 23:28:03 -0700 Subject: [PATCH 14/20] Remove logging options in favor of defaults --- pytest.ini | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pytest.ini b/pytest.ini index 82490cd4..3c583daf 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,7 +3,3 @@ addopts = --verbose --import-mode=importlib testpaths = tensorflow_transform python_files = *_test.py norecursedirs = .* *.egg -log_format = %(asctime)s %(levelname)s %(message)s -log_date_format = %Y-%m-%d %H:%M:%S -log_cli = True -log_cli_level = INFO \ No newline at end of file From f63a78f6eb008e6a0f6ea2cb4ca37889b5e9df32 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Mon, 26 Aug 2024 23:22:24 -0700 Subject: [PATCH 15/20] Remove verbose flag --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 3c583daf..0342498b 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,5 @@ [pytest] -addopts = --verbose --import-mode=importlib +addopts = --import-mode=importlib testpaths = tensorflow_transform python_files = *_test.py norecursedirs = .* *.egg From ffbb959b414305b45716c69b4bd45f715a00b692 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Tue, 27 Aug 2024 00:34:19 -0700 Subject: [PATCH 16/20] Add xfail mark to classes with failing tests --- tensorflow_transform/beam/bucketize_integration_test.py | 4 ++++ tensorflow_transform/beam/impl_output_record_batches_test.py | 4 ++++ tensorflow_transform/beam/impl_test.py | 4 ++++ tensorflow_transform/beam/tukey_hh_params_integration_test.py | 4 ++++ tensorflow_transform/beam/vocabulary_integration_test.py | 4 ++++ .../beam/vocabulary_tfrecord_gzip_integration_test.py | 4 ++++ tensorflow_transform/tf_utils_test.py | 4 ++++ 7 files changed, 28 insertions(+) diff --git a/tensorflow_transform/beam/bucketize_integration_test.py b/tensorflow_transform/beam/bucketize_integration_test.py index 7ce9a62d..2f19ce79 100644 --- a/tensorflow_transform/beam/bucketize_integration_test.py +++ b/tensorflow_transform/beam/bucketize_integration_test.py @@ -13,6 +13,8 @@ # limitations under the License. """Tests for tft.bucketize and tft.quantiles.""" + +import pytest import contextlib import random @@ -339,6 +341,8 @@ def _compute_simple_per_key_bucket(val, key, weighted=False): ] +@pytest.mark.xfail(reason="PR 315 This class contains tests that fail and needs to be fixed. " +"If all tests pass, please remove this mark.") class BucketizeIntegrationTest(tft_unit.TransformTestCase): def setUp(self): diff --git a/tensorflow_transform/beam/impl_output_record_batches_test.py b/tensorflow_transform/beam/impl_output_record_batches_test.py index a2bb2b5f..d83d263f 100644 --- a/tensorflow_transform/beam/impl_output_record_batches_test.py +++ b/tensorflow_transform/beam/impl_output_record_batches_test.py @@ -13,6 +13,8 @@ # limitations under the License. """Same as impl_test.py, except that impl produces `pa.RecordBatch`es.""" + +import pytest import collections import numpy as np @@ -28,6 +30,8 @@ _LARGE_BATCH_SIZE = 1 << 10 +@pytest.mark.xfail(reason="PR 315 This class contains tests that fail and needs to be fixed. " +"If all tests pass, please remove this mark.") class BeamImplOutputRecordBatchesTest(impl_test.BeamImplTest): def _OutputRecordBatches(self): diff --git a/tensorflow_transform/beam/impl_test.py b/tensorflow_transform/beam/impl_test.py index 4cbb9fb0..231b8c5d 100644 --- a/tensorflow_transform/beam/impl_test.py +++ b/tensorflow_transform/beam/impl_test.py @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. + +import pytest import itertools import math import os @@ -110,6 +112,8 @@ def _mean_output_dtype(input_dtype): return tf.float64 if input_dtype == tf.float64 else tf.float32 +@pytest.mark.xfail(reason="PR 315 This class contains tests that fail and needs to be fixed. " +"If all tests pass, please remove this mark.") class BeamImplTest(tft_unit.TransformTestCase): def setUp(self): diff --git a/tensorflow_transform/beam/tukey_hh_params_integration_test.py b/tensorflow_transform/beam/tukey_hh_params_integration_test.py index 05d28a01..23432df5 100644 --- a/tensorflow_transform/beam/tukey_hh_params_integration_test.py +++ b/tensorflow_transform/beam/tukey_hh_params_integration_test.py @@ -13,6 +13,8 @@ # limitations under the License. """Tests for tft.tukey_* calls (Tukey HH parameters).""" + +import pytest import itertools import apache_beam as beam @@ -92,6 +94,8 @@ ] +@pytest.mark.xfail(reason="PR 315 This class contains tests that fail and needs to be fixed. " +"If all tests pass, please remove this mark.") class TukeyHHParamsIntegrationTest(tft_unit.TransformTestCase): def setUp(self): diff --git a/tensorflow_transform/beam/vocabulary_integration_test.py b/tensorflow_transform/beam/vocabulary_integration_test.py index 2899d91a..20f94f3c 100644 --- a/tensorflow_transform/beam/vocabulary_integration_test.py +++ b/tensorflow_transform/beam/vocabulary_integration_test.py @@ -15,6 +15,8 @@ # limitations under the License. """Tests for tft.vocabulary and tft.compute_and_apply_vocabulary.""" + +import pytest import os import apache_beam as beam @@ -114,6 +116,8 @@ ] +@pytest.mark.xfail(reason="PR 315 This class contains tests that fail and needs to be fixed. " +"If all tests pass, please remove this mark.") class VocabularyIntegrationTest(tft_unit.TransformTestCase): def setUp(self): diff --git a/tensorflow_transform/beam/vocabulary_tfrecord_gzip_integration_test.py b/tensorflow_transform/beam/vocabulary_tfrecord_gzip_integration_test.py index 9b49eaa0..25f32b07 100644 --- a/tensorflow_transform/beam/vocabulary_tfrecord_gzip_integration_test.py +++ b/tensorflow_transform/beam/vocabulary_tfrecord_gzip_integration_test.py @@ -14,10 +14,14 @@ # limitations under the License. """Tests for tfrecord_gzip tft.vocabulary and tft.compute_and_apply_vocabulary.""" + +import pytest from tensorflow_transform.beam import vocabulary_integration_test from tensorflow_transform.beam import tft_unit +@pytest.mark.xfail(reason="PR 315 This class contains tests that fail and needs to be fixed. " +"If all tests pass, please remove this mark.") class TFRecordVocabularyIntegrationTest( vocabulary_integration_test.VocabularyIntegrationTest): diff --git a/tensorflow_transform/tf_utils_test.py b/tensorflow_transform/tf_utils_test.py index c296d918..72e1e61b 100644 --- a/tensorflow_transform/tf_utils_test.py +++ b/tensorflow_transform/tf_utils_test.py @@ -13,6 +13,8 @@ # limitations under the License. """Tests for tensorflow_transform.tf_utils.""" + +import pytest import os import numpy as np @@ -65,6 +67,8 @@ def __init__(self, shape, dtype): tf.SparseTensorSpec = _SparseTensorSpec +@pytest.mark.xfail(reason="PR 315 This class contains tests that fail and needs to be fixed. " +"If all tests pass, please remove this mark.") class TFUtilsTest(test_case.TransformTestCase): def _assertCompositeRefEqual(self, left, right): From fbac9aff59f5cd65729957983af23249ef20177e Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Tue, 27 Aug 2024 00:56:37 -0700 Subject: [PATCH 17/20] Remove timeout --- .github/workflows/ci-test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index e9ea140a..6180b2d5 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -13,7 +13,6 @@ jobs: unit-tests: if: github.actor != 'copybara-service[bot]' runs-on: ubuntu-latest - timeout-minutes: 60 strategy: matrix: From 82466a258d6a9c580e630c8d822abe7237b1dd26 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Tue, 27 Aug 2024 01:09:32 -0700 Subject: [PATCH 18/20] Don't run xfailed tests --- tensorflow_transform/beam/bucketize_integration_test.py | 2 +- tensorflow_transform/beam/impl_output_record_batches_test.py | 2 +- tensorflow_transform/beam/impl_test.py | 2 +- tensorflow_transform/beam/tukey_hh_params_integration_test.py | 2 +- tensorflow_transform/beam/vocabulary_integration_test.py | 2 +- .../beam/vocabulary_tfrecord_gzip_integration_test.py | 2 +- tensorflow_transform/tf_utils_test.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tensorflow_transform/beam/bucketize_integration_test.py b/tensorflow_transform/beam/bucketize_integration_test.py index 2f19ce79..4d7fa3b9 100644 --- a/tensorflow_transform/beam/bucketize_integration_test.py +++ b/tensorflow_transform/beam/bucketize_integration_test.py @@ -341,7 +341,7 @@ def _compute_simple_per_key_bucket(val, key, weighted=False): ] -@pytest.mark.xfail(reason="PR 315 This class contains tests that fail and needs to be fixed. " +@pytest.mark.xfail(run=False, reason="PR 315 This class contains tests that fail and needs to be fixed. " "If all tests pass, please remove this mark.") class BucketizeIntegrationTest(tft_unit.TransformTestCase): diff --git a/tensorflow_transform/beam/impl_output_record_batches_test.py b/tensorflow_transform/beam/impl_output_record_batches_test.py index d83d263f..4e5126a3 100644 --- a/tensorflow_transform/beam/impl_output_record_batches_test.py +++ b/tensorflow_transform/beam/impl_output_record_batches_test.py @@ -30,7 +30,7 @@ _LARGE_BATCH_SIZE = 1 << 10 -@pytest.mark.xfail(reason="PR 315 This class contains tests that fail and needs to be fixed. " +@pytest.mark.xfail(run=False, reason="PR 315 This class contains tests that fail and needs to be fixed. " "If all tests pass, please remove this mark.") class BeamImplOutputRecordBatchesTest(impl_test.BeamImplTest): diff --git a/tensorflow_transform/beam/impl_test.py b/tensorflow_transform/beam/impl_test.py index 231b8c5d..3cb9b619 100644 --- a/tensorflow_transform/beam/impl_test.py +++ b/tensorflow_transform/beam/impl_test.py @@ -112,7 +112,7 @@ def _mean_output_dtype(input_dtype): return tf.float64 if input_dtype == tf.float64 else tf.float32 -@pytest.mark.xfail(reason="PR 315 This class contains tests that fail and needs to be fixed. " +@pytest.mark.xfail(run=False, reason="PR 315 This class contains tests that fail and needs to be fixed. " "If all tests pass, please remove this mark.") class BeamImplTest(tft_unit.TransformTestCase): diff --git a/tensorflow_transform/beam/tukey_hh_params_integration_test.py b/tensorflow_transform/beam/tukey_hh_params_integration_test.py index 23432df5..70ff05e0 100644 --- a/tensorflow_transform/beam/tukey_hh_params_integration_test.py +++ b/tensorflow_transform/beam/tukey_hh_params_integration_test.py @@ -94,7 +94,7 @@ ] -@pytest.mark.xfail(reason="PR 315 This class contains tests that fail and needs to be fixed. " +@pytest.mark.xfail(run=False, reason="PR 315 This class contains tests that fail and needs to be fixed. " "If all tests pass, please remove this mark.") class TukeyHHParamsIntegrationTest(tft_unit.TransformTestCase): diff --git a/tensorflow_transform/beam/vocabulary_integration_test.py b/tensorflow_transform/beam/vocabulary_integration_test.py index 20f94f3c..0fbfda42 100644 --- a/tensorflow_transform/beam/vocabulary_integration_test.py +++ b/tensorflow_transform/beam/vocabulary_integration_test.py @@ -116,7 +116,7 @@ ] -@pytest.mark.xfail(reason="PR 315 This class contains tests that fail and needs to be fixed. " +@pytest.mark.xfail(run=False, reason="PR 315 This class contains tests that fail and needs to be fixed. " "If all tests pass, please remove this mark.") class VocabularyIntegrationTest(tft_unit.TransformTestCase): diff --git a/tensorflow_transform/beam/vocabulary_tfrecord_gzip_integration_test.py b/tensorflow_transform/beam/vocabulary_tfrecord_gzip_integration_test.py index 25f32b07..1faee893 100644 --- a/tensorflow_transform/beam/vocabulary_tfrecord_gzip_integration_test.py +++ b/tensorflow_transform/beam/vocabulary_tfrecord_gzip_integration_test.py @@ -20,7 +20,7 @@ from tensorflow_transform.beam import tft_unit -@pytest.mark.xfail(reason="PR 315 This class contains tests that fail and needs to be fixed. " +@pytest.mark.xfail(run=False, reason="PR 315 This class contains tests that fail and needs to be fixed. " "If all tests pass, please remove this mark.") class TFRecordVocabularyIntegrationTest( vocabulary_integration_test.VocabularyIntegrationTest): diff --git a/tensorflow_transform/tf_utils_test.py b/tensorflow_transform/tf_utils_test.py index 72e1e61b..14b60f7f 100644 --- a/tensorflow_transform/tf_utils_test.py +++ b/tensorflow_transform/tf_utils_test.py @@ -67,7 +67,7 @@ def __init__(self, shape, dtype): tf.SparseTensorSpec = _SparseTensorSpec -@pytest.mark.xfail(reason="PR 315 This class contains tests that fail and needs to be fixed. " +@pytest.mark.xfail(run=False, reason="PR 315 This class contains tests that fail and needs to be fixed. " "If all tests pass, please remove this mark.") class TFUtilsTest(test_case.TransformTestCase): From 46f07167d961a2b16b1664282fa36b7ee78fdce4 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Thu, 3 Jul 2025 10:11:48 -0700 Subject: [PATCH 19/20] Add `xfail` mark to failing test --- tensorflow_transform/beam/analysis_graph_builder_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorflow_transform/beam/analysis_graph_builder_test.py b/tensorflow_transform/beam/analysis_graph_builder_test.py index 772b6927..89d00684 100644 --- a/tensorflow_transform/beam/analysis_graph_builder_test.py +++ b/tensorflow_transform/beam/analysis_graph_builder_test.py @@ -16,6 +16,7 @@ import os import sys +import pytest import tensorflow as tf import tensorflow_transform as tft from tensorflow_transform import analyzer_nodes @@ -412,6 +413,8 @@ class AnalysisGraphBuilderTest(tft_unit.TransformTestCase): ], ) ) + @pytest.mark.xfail(run=False, reason="PR 315 This class contains tests that fail and needs to be fixed. " + "If all tests pass, please remove this mark.") def test_build( self, feature_spec, From 3676584620d9dee74c61d5d35e46a4c95722d2b3 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Fri, 4 Jul 2025 14:24:12 -0700 Subject: [PATCH 20/20] Undo ruff formatting --- setup.py | 170 +++++++++++++++++++++++++++---------------------------- 1 file changed, 83 insertions(+), 87 deletions(-) diff --git a/setup.py b/setup.py index 181d3906..456ecc61 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. """Package Setup script for tf.Transform.""" - import os from pathlib import Path @@ -21,112 +20,109 @@ def select_constraint(default, nightly=None, git_master=None): - """Select dependency constraint based on TFX_DEPENDENCY_SELECTOR env var.""" - selector = os.environ.get("TFX_DEPENDENCY_SELECTOR") - if selector == "UNCONSTRAINED": - return "" - elif selector == "NIGHTLY" and nightly is not None: - return nightly - elif selector == "GIT_MASTER" and git_master is not None: - return git_master - else: - return default + """Select dependency constraint based on TFX_DEPENDENCY_SELECTOR env var.""" + selector = os.environ.get('TFX_DEPENDENCY_SELECTOR') + if selector == 'UNCONSTRAINED': + return '' + elif selector == 'NIGHTLY' and nightly is not None: + return nightly + elif selector == 'GIT_MASTER' and git_master is not None: + return git_master + else: + return default # Get version from version module. -with open("tensorflow_transform/version.py") as fp: - globals_dict = {} - exec(fp.read(), globals_dict) # pylint: disable=exec-used -__version__ = globals_dict["__version__"] +with open('tensorflow_transform/version.py') as fp: + globals_dict = {} + exec(fp.read(), globals_dict) # pylint: disable=exec-used +__version__ = globals_dict['__version__'] def _make_required_install_packages(): - # Make sure to sync the versions of common dependencies (absl-py, numpy, and - # protobuf) with TF and pyarrow version with tfx-bsl. - return [ - "absl-py>=0.9,<2.0.0", - 'apache-beam[gcp]>=2.53,<3;python_version>="3.11"', - 'apache-beam[gcp]>=2.50,<2.51;python_version<"3.11"', - "numpy>=1.22.0", - 'protobuf>=4.25.2,<6.0.0;python_version>="3.11"', - 'protobuf>=4.21.6,<6.0.0;python_version<"3.11"', - "pyarrow>=10,<11", - "pydot>=1.2,<2", - "tensorflow>=2.17,<2.18", - "tensorflow-metadata" - + select_constraint( - default=">=1.17.1,<1.18.0", - nightly=">=1.18.0.dev", - git_master="@git+https://github.com/tensorflow/metadata@master", - ), - "tf_keras>=2", - "tfx-bsl" - + select_constraint( - default=">=1.17.1,<1.18.0", - nightly=">=1.18.0.dev", - git_master="@git+https://github.com/tensorflow/tfx-bsl@master", - ), - ] - + # Make sure to sync the versions of common dependencies (absl-py, numpy, and + # protobuf) with TF and pyarrow version with tfx-bsl. + return [ + 'absl-py>=0.9,<2.0.0', + 'apache-beam[gcp]>=2.53,<3;python_version>="3.11"', + 'apache-beam[gcp]>=2.50,<2.51;python_version<"3.11"', + 'numpy>=1.22.0', + 'protobuf>=4.25.2,<6.0.0;python_version>="3.11"', + 'protobuf>=4.21.6,<6.0.0;python_version<"3.11"', + 'pyarrow>=10,<11', + 'pydot>=1.2,<2', + 'tensorflow>=2.17,<2.18', + 'tensorflow-metadata' + + select_constraint( + default='>=1.17.1,<1.18.0', + nightly='>=1.18.0.dev', + git_master='@git+https://github.com/tensorflow/metadata@master', + ), + 'tf_keras>=2', + 'tfx-bsl' + + select_constraint( + default='>=1.17.1,<1.18.0', + nightly='>=1.18.0.dev', + git_master='@git+https://github.com/tensorflow/tfx-bsl@master', + ), + ] def _make_docs_packages(): - return [ - req - for req in Path("./requirements-docs.txt") - .expanduser() - .resolve() - .read_text() - .splitlines() - if req - ] + return [ + req for req in Path("./requirements-docs.txt") + .expanduser() + .resolve() + .read_text() + .splitlines() + if req + ] # Get the long description from the README file. -with open("README.md") as fp: - _LONG_DESCRIPTION = fp.read() +with open('README.md') as fp: + _LONG_DESCRIPTION = fp.read() setup( - name="tensorflow-transform", + name='tensorflow-transform', version=__version__, - author="Google Inc.", - author_email="tensorflow-extended-dev@googlegroups.com", - license="Apache 2.0", + author='Google Inc.', + author_email='tensorflow-extended-dev@googlegroups.com', + license='Apache 2.0', classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "Intended Audience :: Education", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: Apache Software License", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3 :: Only", - "Topic :: Scientific/Engineering", - "Topic :: Scientific/Engineering :: Artificial Intelligence", - "Topic :: Scientific/Engineering :: Mathematics", - "Topic :: Software Development", - "Topic :: Software Development :: Libraries", - "Topic :: Software Development :: Libraries :: Python Modules", + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Intended Audience :: Education', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3 :: Only', + 'Topic :: Scientific/Engineering', + 'Topic :: Scientific/Engineering :: Artificial Intelligence', + 'Topic :: Scientific/Engineering :: Mathematics', + 'Topic :: Software Development', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', ], namespace_packages=[], install_requires=_make_required_install_packages(), - extras_require={ - "test": ["pytest>=8.0"], - "docs": _make_docs_packages(), + extras_require= { + 'test': ['pytest>=8.0'], + 'docs': _make_docs_packages(), }, - python_requires=">=3.9,<4", + python_requires='>=3.9,<4', packages=find_packages(), include_package_data=True, - package_data={"tensorflow_transform": ["py.typed"]}, + package_data={'tensorflow_transform': ['py.typed']}, data_files=[("docs_reqs", ["requirements-docs.txt"])], - description="A library for data preprocessing with TensorFlow", + description='A library for data preprocessing with TensorFlow', long_description=_LONG_DESCRIPTION, - long_description_content_type="text/markdown", - keywords="tensorflow transform tfx", - url="https://www.tensorflow.org/tfx/transform/get_started", - download_url="https://github.com/tensorflow/transform/tags", - requires=[], -) + long_description_content_type='text/markdown', + keywords='tensorflow transform tfx', + url='https://www.tensorflow.org/tfx/transform/get_started', + download_url='https://github.com/tensorflow/transform/tags', + requires=[])