Renamed contrib to sockeye_contrib (#537)

* Renamed contrib to sockeye_contrib

* Typo fix.

.travis.yml

@@ -30,7 +30,7 @@ script:
   - if [ "$TRAVIS_EVENT_TYPE" != "cron" ]; then python -m pytest -k "Copy:transformer:transformer" --maxfail=1 test/system; fi
   - if [ "$TRAVIS_EVENT_TYPE" != "cron" ]; then python -m pytest -k "Copy:cnn:cnn" --maxfail=1 test/system; fi
   - if [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then python -m pytest --maxfail=1 test/system; fi
-  - if [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then python -m contrib.autopilot.test; fi
+  - if [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then python -m sockeye_contrib.autopilot.test; fi
 
 # Deploy all tagged versions to PyPI
 deploy:

CHANGELOG.md

@@ -10,6 +10,10 @@ Note that Sockeye has checks in place to not translate with an old model that wa
 
 Each version section may have have subsections for: _Added_, _Changed_, _Removed_, _Deprecated_, and _Fixed_.
 
+## [1.18.55]
+### Changed
+- Renamed `contrib` to less-generic `sockeye_contrib`
+
 ## [1.18.54]
 ### Added
 - `--source-factor-vocabs` can be set to provide source factor vocabularies.

MANIFEST.in

@@ -10,8 +10,8 @@ include sockeye/git_version.py
 recursive-include .github *
 include CONTRIBUTING.md
 exclude *.sh
-exclude contrib
-recursive-exclude contrib *
+exclude sockeye_contrib
+recursive-exclude sockeye_contrib *
 include pytest.ini
 recursive-include test *.py
 recursive-include test *.ini

README.md

@@ -133,7 +133,7 @@ For example *sockeye-train* can also be invoked as
 
 ## First Steps
 
-For easily training popular model types on known data sets, see the [Sockeye Autopilot documentation](https://github.com/awslabs/sockeye/tree/master/contrib/autopilot).
+For easily training popular model types on known data sets, see the [Sockeye Autopilot documentation](https://github.com/awslabs/sockeye/tree/master/sockeye_contrib/autopilot).
 For manually training and running translation models on your data, read on.
 
 ### Train

sockeye/__init__.py

@@ -11,4 +11,4 @@
 # express or implied. See the License for the specific language governing
 # permissions and limitations under the License.
 
-__version__ = '1.18.54'
+__version__ = '1.18.55'

sockeye/evaluate.py

@@ -23,7 +23,7 @@
 
 import numpy as np
 
-from contrib import sacrebleu, rouge
+from sockeye_contrib import sacrebleu, rouge
 from . import arguments
 from . import constants as C
 from . import data_io

sockeye/lexicon.py

@@ -241,8 +241,8 @@ def main():
     subparams = params.add_subparsers(title="Commands")
 
     params_create = subparams.add_parser('create', description="Create top-k lexicon for use during decoding. "
-                                                               "See contrib/fast_align/README.md for information "
-                                                               "on creating input lexical tables.")
+                                                               "See sockeye_contrib/fast_align/README.md "
+                                                               "for information on creating input lexical tables.")
     arguments.add_lexicon_args(params_create)
     arguments.add_lexicon_create_args(params_create)
     arguments.add_logging_args(params_create)

sockeye/rerank.py

@@ -26,7 +26,7 @@
 from . import utils
 from . import arguments
 from . import constants as C
-from contrib import sacrebleu
+from sockeye_contrib import sacrebleu
 
 logger = log.setup_main_logger(__name__, console=True, file_logging=False)
 

contrib/autopilot/README.md → sockeye_contrib/autopilot/README.md

@@ -15,7 +15,7 @@ If Sockeye is installed via pip or source, Autopilot can be run directly:
 This is equivalent to:
 
 ```bash
-> python -m contrib.autopilot.autopilot
+> python -m sockeye_contrib.autopilot.autopilot
 ```
 
 With a single command, Autopilot can download and pre-process training data, then train and evaluate a translation model.
@@ -129,4 +129,4 @@ Provide the following outputs to the user under "results":
 
 - test.N.MODEL.SETTINGS.bpe.bleu: BLEU score of raw decoder output against byte-pair encoded references
 - test.N.MODEL.SETTINGS.tok.bleu: BLEU score of word-level decoder output against tokenized references
-- test.N.MODEL.SETTINGS.detok.sacrebleu: BLEU score of detokenized decoder output against raw references using [SacreBLEU](https://github.com/awslabs/sockeye/tree/master/contrib/sacrebleu).  These scores are directly comparable to those reported in WMT evaluations.
+- test.N.MODEL.SETTINGS.detok.sacrebleu: BLEU score of detokenized decoder output against raw references using [SacreBLEU](https://github.com/awslabs/sockeye/tree/master/sockeye_contrib/sacrebleu).  These scores are directly comparable to those reported in WMT evaluations.

contrib/autopilot/__init__.py → sockeye_contrib/autopilot/__init__.py

@@ -11,7 +11,7 @@
 # express or implied. See the License for the specific language governing
 # permissions and limitations under the License.
 
-from contrib.autopilot import autopilot
-from contrib.autopilot import tasks
-from contrib.autopilot import models
-from contrib.autopilot import third_party
+from sockeye_contrib.autopilot import autopilot
+from sockeye_contrib.autopilot import tasks
+from sockeye_contrib.autopilot import models
+from sockeye_contrib.autopilot import third_party

contrib/autopilot/autopilot.py → sockeye_contrib/autopilot/autopilot.py

@@ -37,15 +37,15 @@
     raise RuntimeError("Please install the sockeye module or add the sockeye root directory to your Python path. Ex: export PYTHONPATH=%s"
                        % SOCKEYE_ROOT)
 
-from contrib.autopilot.tasks import ARCHIVE_NONE, ARCHIVE_TAR, ARCHIVE_ZIP
-from contrib.autopilot.tasks import TEXT_UTF8_RAW, TEXT_UTF8_RAW_SGML, TEXT_UTF8_RAW_BITEXT
-from contrib.autopilot.tasks import TEXT_UTF8_RAW_BITEXT_REVERSE, TEXT_REQUIRES_TOKENIZATION
-from contrib.autopilot.tasks import TEXT_UTF8_TOKENIZED
-from contrib.autopilot.tasks import RAW_FILES
-from contrib.autopilot.tasks import Task, TASKS
-from contrib.autopilot.models import MODELS, MODEL_NONE, MODEL_GNMT, MODEL_TEST_ARGS
-from contrib.autopilot.models import DECODE_ARGS, DECODE_STANDARD, DECODE_GNMT
-from contrib.autopilot import third_party
+from sockeye_contrib.autopilot.tasks import ARCHIVE_NONE, ARCHIVE_TAR, ARCHIVE_ZIP
+from sockeye_contrib.autopilot.tasks import TEXT_UTF8_RAW, TEXT_UTF8_RAW_SGML, TEXT_UTF8_RAW_BITEXT
+from sockeye_contrib.autopilot.tasks import TEXT_UTF8_RAW_BITEXT_REVERSE, TEXT_REQUIRES_TOKENIZATION
+from sockeye_contrib.autopilot.tasks import TEXT_UTF8_TOKENIZED
+from sockeye_contrib.autopilot.tasks import RAW_FILES
+from sockeye_contrib.autopilot.tasks import Task, TASKS
+from sockeye_contrib.autopilot.models import MODELS, MODEL_NONE, MODEL_GNMT, MODEL_TEST_ARGS
+from sockeye_contrib.autopilot.models import DECODE_ARGS, DECODE_STANDARD, DECODE_GNMT
+from sockeye_contrib.autopilot import third_party
 
 
 # Formats for custom files
@@ -457,7 +457,7 @@ def call_sockeye_translate(args: List[str],
 
 def call_sacrebleu(input_fname: str, ref_fname: str, output_fname: str, log_fname: str, tokenized: bool = False):
     """
-    Call contrib.sacrebleu.sacrebleu on tokenized or detokenized inputs.
+    Call sockeye_contrib.sacrebleu.sacrebleu on tokenized or detokenized inputs.
 
     :param input_fname: Input translation file.
     :param ref_fname: Reference translation file.
@@ -468,7 +468,7 @@ def call_sacrebleu(input_fname: str, ref_fname: str, output_fname: str, log_fnam
     # Assemble command
     command = [sys.executable,
                "-m",
-               "contrib.sacrebleu.sacrebleu",
+               "sockeye_contrib.sacrebleu.sacrebleu",
                "--score-only",
                "--input={}".format(input_fname),
                ref_fname]
@@ -477,7 +477,7 @@ def call_sacrebleu(input_fname: str, ref_fname: str, output_fname: str, log_fnam
         command.append("--tokenize=none")
     # Call sacrebleu
     with open(log_fname, "wb") as log:
-        logging.info("contrib.sacrebleu.sacrebleu: %s -> %s", input_fname, output_fname)
+        logging.info("sockeye_contrib.sacrebleu.sacrebleu: %s -> %s", input_fname, output_fname)
         logging.info("Log: %s", log_fname)
         score = subprocess.check_output(command, stderr=log)
     # Record successful score
@@ -796,7 +796,7 @@ def report_data(file_sets):
         else:
             fname_log = os.path.join(args.workspace,
                          DIR_LOGS,
-                         "contrib.sacrebleu.sacrebleu.{}.{}.{}.{}.log".format(task_name,
+                         "sockeye_contrib.sacrebleu.sacrebleu.{}.{}.{}.{}.log".format(task_name,
                                                                               args.model,
                                                                               fname_base + SUFFIX_BPE,
                                                                               os.getpid()))
@@ -817,7 +817,7 @@ def report_data(file_sets):
             third_party.merge_bpe(input_fname=fname_bpe, output_fname=fname_tok)
             fname_log = os.path.join(args.workspace,
                          DIR_LOGS,
-                         "contrib.sacrebleu.sacrebleu.{}.{}.{}.{}.log".format(task_name,
+                         "sockeye_contrib.sacrebleu.sacrebleu.{}.{}.{}.{}.log".format(task_name,
                                                                               args.model,
                                                                               fname_base + SUFFIX_TOK,
                                                                               os.getpid()))
@@ -844,7 +844,7 @@ def report_data(file_sets):
                                                lang_code=lang_code)
             fname_log = os.path.join(args.workspace,
                          DIR_LOGS,
-                         "contrib.sacrebleu.sacrebleu.{}.{}.{}.{}.log".format(task_name,
+                         "sockeye_contrib.sacrebleu.sacrebleu.{}.{}.{}.{}.log".format(task_name,
                                                                               args.model,
                                                                               fname_base + SUFFIX_DETOK,
                                                                               os.getpid()))

contrib/autopilot/test.py → sockeye_contrib/autopilot/test.py

@@ -23,7 +23,7 @@
 
 # Make sure the version of sockeye being tested is first on the system path
 try:
-    import contrib.autopilot.autopilot as autopilot
+    import sockeye_contrib.autopilot.autopilot as autopilot
 except ImportError:
     SOCKEYE_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
     PYTHONPATH = "PYTHONPATH"
@@ -32,7 +32,7 @@
     else:
         os.environ[PYTHONPATH] = SOCKEYE_ROOT
     sys.path.append(SOCKEYE_ROOT)
-    import contrib.autopilot.autopilot as autopilot
+    import sockeye_contrib.autopilot.autopilot as autopilot
 
 
 # Test-specific constants
@@ -84,7 +84,7 @@ def main():
         # WMT task with raw data (Transformer)
         command = [sys.executable,
                    "-m",
-                   "contrib.autopilot.autopilot",
+                   "sockeye_contrib.autopilot.autopilot",
                    "--task={}".format(WMT_TASK),
                    "--model=transformer",
                    "--gpus=0",
@@ -94,7 +94,7 @@ def main():
         # WMT task with raw data (GNMT)
         command = [sys.executable,
                    "-m",
-                   "contrib.autopilot.autopilot",
+                   "sockeye_contrib.autopilot.autopilot",
                    "--task={}".format(WMT_TASK),
                    "--model=gnmt_like",
                    "--decode-settings=gnmt_like",
@@ -107,7 +107,7 @@ def main():
         # WNMT task with pre-tokenized data (Transformer)
         # command = [sys.executable,
         #            "-m",
-        #            "contrib.autopilot.autopilot",
+        #            "sockeye_contrib.autopilot.autopilot",
         #            "--task={}".format(WNMT_TASK),
         #            "--model=transformer",
         #            "--gpus=0",
@@ -117,7 +117,7 @@ def main():
         # WMT task, prepare data only
         command = [sys.executable,
                    "-m",
-                   "contrib.autopilot.autopilot",
+                   "sockeye_contrib.autopilot.autopilot",
                    "--task={}".format(DATA_ONLY_TASK),
                    "--model=none",
                    "--gpus=0",
@@ -127,7 +127,7 @@ def main():
         # Custom task (raw data, Transformer)
         command = [sys.executable,
                    "-m",
-                   "contrib.autopilot.autopilot",
+                   "sockeye_contrib.autopilot.autopilot",
                    "--custom-task=custom_raw",
                    "--custom-train",
                    os.path.join(work_dir, autopilot.DIR_SYSTEMS, WMT_TASK + autopilot.SUFFIX_TEST, autopilot.DIR_DATA,
@@ -156,7 +156,7 @@ def main():
         # Custom task (tokenized data, Transformer)
         command = [sys.executable,
                    "-m",
-                   "contrib.autopilot.autopilot",
+                   "sockeye_contrib.autopilot.autopilot",
                    "--custom-task=custom_tok",
                    "--custom-train",
                    os.path.join(work_dir, autopilot.DIR_SYSTEMS, WMT_TASK + autopilot.SUFFIX_TEST, autopilot.DIR_DATA,
@@ -183,7 +183,7 @@ def main():
         # Custom task (byte-pair encoded data, Transformer)
         command = [sys.executable,
                    "-m",
-                   "contrib.autopilot.autopilot",
+                   "sockeye_contrib.autopilot.autopilot",
                    "--custom-task=custom_bpe",
                    "--custom-train",
                    os.path.join(work_dir, autopilot.DIR_SYSTEMS, WMT_TASK + autopilot.SUFFIX_TEST, autopilot.DIR_DATA,

contrib/sacrebleu/README.md → sockeye_contrib/sacrebleu/README.md

@@ -73,4 +73,4 @@ SacreBLEU is licensed under the Apache 2.0 License.
 
 This was all Rico Sennrich's idea.
 Originally written by Matt Post.
-The official version can be found at https://github.com/awslabs/sockeye/tree/master/contrib/sacrebleu.
+The official version can be found at https://github.com/awslabs/sockeye/tree/master/sockeye_contrib/sacrebleu.

contrib/sacrebleu/__init__.py → sockeye_contrib/sacrebleu/__init__.py

@@ -11,4 +11,4 @@
 # express or implied. See the License for the specific language governing
 # permissions and limitations under the License.
 
-from contrib.sacrebleu.sacrebleu import raw_corpus_bleu, compute_bleu, corpus_chrf, CHRF_ORDER, CHRF_BETA, sentence_bleu, sentence_chrf
+from sockeye_contrib.sacrebleu.sacrebleu import raw_corpus_bleu, compute_bleu, corpus_chrf, CHRF_ORDER, CHRF_BETA, sentence_bleu, sentence_chrf

contrib/vistools/README.md → sockeye_contrib/vistools/README.md

@@ -7,7 +7,7 @@ This package generates javascript-based graphs of the beam search performed by s
 First, install the dependencies required for the visualizations:
 
 ```sh
-pip install -r contrib/vistools/requirements.txt
+pip install -r sockeye_contrib/vistools/requirements.txt
 ```
 
 ### Store the beam histories
@@ -27,7 +27,7 @@ python3 -m sockeye.translate --models model \
 After inference, the graphs can be generated with:
 
 ```
-python3 contrib/vistools/generate_graphs.py -d beams.json -o generated_graphs
+python3 sockeye_contrib/vistools/generate_graphs.py -d beams.json -o generated_graphs
 ```
 
 The `generated_graphs/` folder will contain one `html` file per sentence. Opening it in your browser will show the interactive graph.

test/unit/test_bleu.py

@@ -15,7 +15,7 @@
 
 import pytest
 
-from contrib import sacrebleu
+from sockeye_contrib import sacrebleu
 
 EPSILON = 1e-8
 

test/unit/test_chrf.py

@@ -15,7 +15,7 @@
 
 import pytest
 
-from contrib import sacrebleu
+from sockeye_contrib import sacrebleu
 
 EPSILON = 1e-8
 

test/unit/test_rouge.py

@@ -13,7 +13,7 @@
 
 import pytest
 
-from contrib import rouge
+from sockeye_contrib import rouge
 
 test_cases = [(["this is a test", "another test case"], ["this is a test case", "another test case"], 0.9444444394753087, 0.928571423622449, 0.9338624338620563),
               (["this is a single test case"], ["this is a single test case"], 0.999999995, 0.999999995, 0.9999999999995),