Skip to content

Commit 56aa954

Browse files
mthrokmthrok
authored
Add OPUS support to 0.6 release (#763)
* Use cmake for third party (#753) * Use cmake for third party * Apply patch to libmad * Update gitignore * Update docker test image * Add opus support (#755)
1 parent ee98d07 commit 56aa954

File tree

21 files changed

+332
-402
lines changed

21 files changed

+332
-402
lines changed

‎.circleci/config.yml

Lines changed: 27 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,13 @@ commands:
3838
our_upload_channel=test
3939
fi
4040
echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
41+
install_build_tools_macos:
42+
description: "installs tools required to build torchaudio"
43+
steps:
44+
- run:
45+
name: Install cmake and pkg-config
46+
command: HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake pkg-config
47+
# Disable brew auto update which is very slow
4148

4249
binary_common: &binary_common
4350
parameters:
@@ -82,28 +89,30 @@ jobs:
8289
8390
download_third_parties_nix:
8491
docker:
85-
- image: "centos:8"
92+
- image: "pytorch/torchaudio_unittest_base:manylinux"
8693
resource_class: small
8794
steps:
8895
- checkout
8996
- generate_cache_key
9097
- restore_cache:
9198

9299
keys:
93-
- tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
100+
- tp-nix-v2-{{ checksum ".cachekey" }}
94101

95102
- run:
96-
command: ./build_tools/setup_helpers/build_third_party.sh $PWD --download-only
103+
command: |
104+
mkdir -p third_party/archives/
105+
wget --no-clobber --directory-prefix=third_party/archives/ $(awk '/URL /{print $2}' third_party/CMakeLists.txt)
97106
- save_cache:
98107

99-
key: tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
108+
key: tp-nix-v2-{{ checksum ".cachekey" }}
100109

101110
paths:
102-
- third_party/tmp
111+
- third_party/archives
103112
- persist_to_workspace:
104113
root: third_party
105114
paths:
106-
- tmp
115+
- archives
107116

108117
binary_linux_wheel:
109118
<<: *binary_common
@@ -145,6 +154,7 @@ jobs:
145154
xcode: "9.0"
146155
steps:
147156
- checkout
157+
- install_build_tools_macos
148158
- attach_workspace:
149159
at: third_party
150160
- run:
@@ -169,6 +179,7 @@ jobs:
169179
xcode: "9.0"
170180
steps:
171181
- checkout
182+
- install_build_tools_macos
172183
- attach_workspace:
173184
at: third_party
174185
- run:
@@ -383,19 +394,21 @@ jobs:
383394
- restore_cache:
384395

385396
keys:
386-
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
397+
- env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
387398

388399
- run:
389400
name: Setup
390401
command: .circleci/unittest/linux/scripts/setup_env.sh
391402
- save_cache:
392403

393-
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
404+
key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
394405

395406
paths:
396407
- conda
397408
- env
398409
- third_party/build
410+
- third_party/install
411+
- third_party/src
399412
- run:
400413
name: Install torchaudio
401414
command: .circleci/unittest/linux/scripts/install.sh
@@ -423,19 +436,21 @@ jobs:
423436
- restore_cache:
424437

425438
keys:
426-
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
439+
- env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
427440

428441
- run:
429442
name: Setup
430443
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
431444
- save_cache:
432445

433-
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
446+
key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
434447

435448
paths:
436449
- conda
437450
- env
438451
- third_party/build
452+
- third_party/install
453+
- third_party/src
439454
- run:
440455
name: Install torchaudio
441456
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
@@ -542,6 +557,8 @@ jobs:
542557
- conda
543558
- env
544559
- third_party/build
560+
- third_party/install
561+
- third_party/src
545562
- run:
546563
name: Run style check
547564
command: .circleci/unittest/linux/scripts/run_style_checks.sh

‎.circleci/config.yml.in

Lines changed: 27 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,13 @@ commands:
3838
our_upload_channel=test
3939
fi
4040
echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
41+
install_build_tools_macos:
42+
description: "installs tools required to build torchaudio"
43+
steps:
44+
- run:
45+
name: Install cmake and pkg-config
46+
command: HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake pkg-config
47+
# Disable brew auto update which is very slow
4148

4249
binary_common: &binary_common
4350
parameters:
@@ -82,28 +89,30 @@ jobs:
8289

8390
download_third_parties_nix:
8491
docker:
85-
- image: "centos:8"
92+
- image: "pytorch/torchaudio_unittest_base:manylinux"
8693
resource_class: small
8794
steps:
8895
- checkout
8996
- generate_cache_key
9097
- restore_cache:
9198
{% raw %}
9299
keys:
93-
- tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
100+
- tp-nix-v2-{{ checksum ".cachekey" }}
94101
{% endraw %}
95102
- run:
96-
command: ./build_tools/setup_helpers/build_third_party.sh $PWD --download-only
103+
command: |
104+
mkdir -p third_party/archives/
105+
wget --no-clobber --directory-prefix=third_party/archives/ $(awk '/URL /{print $2}' third_party/CMakeLists.txt)
97106
- save_cache:
98107
{% raw %}
99-
key: tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
108+
key: tp-nix-v2-{{ checksum ".cachekey" }}
100109
{% endraw %}
101110
paths:
102-
- third_party/tmp
111+
- third_party/archives
103112
- persist_to_workspace:
104113
root: third_party
105114
paths:
106-
- tmp
115+
- archives
107116

108117
binary_linux_wheel:
109118
<<: *binary_common
@@ -145,6 +154,7 @@ jobs:
145154
xcode: "9.0"
146155
steps:
147156
- checkout
157+
- install_build_tools_macos
148158
- attach_workspace:
149159
at: third_party
150160
- run:
@@ -169,6 +179,7 @@ jobs:
169179
xcode: "9.0"
170180
steps:
171181
- checkout
182+
- install_build_tools_macos
172183
- attach_workspace:
173184
at: third_party
174185
- run:
@@ -383,19 +394,21 @@ jobs:
383394
- restore_cache:
384395
{% raw %}
385396
keys:
386-
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
397+
- env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
387398
{% endraw %}
388399
- run:
389400
name: Setup
390401
command: .circleci/unittest/linux/scripts/setup_env.sh
391402
- save_cache:
392403
{% raw %}
393-
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
404+
key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
394405
{% endraw %}
395406
paths:
396407
- conda
397408
- env
398409
- third_party/build
410+
- third_party/install
411+
- third_party/src
399412
- run:
400413
name: Install torchaudio
401414
command: .circleci/unittest/linux/scripts/install.sh
@@ -423,19 +436,21 @@ jobs:
423436
- restore_cache:
424437
{% raw %}
425438
keys:
426-
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
439+
- env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
427440
{% endraw %}
428441
- run:
429442
name: Setup
430443
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
431444
- save_cache:
432445
{% raw %}
433-
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
446+
key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
434447
{% endraw %}
435448
paths:
436449
- conda
437450
- env
438451
- third_party/build
452+
- third_party/install
453+
- third_party/src
439454
- run:
440455
name: Install torchaudio
441456
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
@@ -542,6 +557,8 @@ jobs:
542557
- conda
543558
- env
544559
- third_party/build
560+
- third_party/install
561+
- third_party/src
545562
- run:
546563
name: Run style check
547564
command: .circleci/unittest/linux/scripts/run_style_checks.sh

‎.circleci/unittest/linux/docker/Dockerfile

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,8 @@ RUN apt update && apt install -y \
5858
sox \
5959
libsox-dev \
6060
libsox-fmt-all \
61+
cmake \
62+
pkg-config \
6163
&& rm -rf /var/lib/apt/lists/*
6264
COPY --from=builder /kaldi /kaldi
6365
COPY --from=builder /third_party /third_party

‎.circleci/unittest/linux/scripts/run_test.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,5 @@ eval "$(./conda/bin/conda shell.bash hook)"
66
conda activate ./env
77

88
python -m torch.utils.collect_env
9-
export PATH="${PWD}/third_party/build/bin/:${PATH}"
9+
export PATH="${PWD}/third_party/install/bin/:${PATH}"
1010
pytest --cov=torchaudio --junitxml=test-results/junit.xml -v --durations 20 test

‎.circleci/unittest/linux/scripts/setup_env.sh

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,5 +33,10 @@ conda activate "${env_dir}"
3333
printf "* Installing dependencies (except PyTorch)\n"
3434
conda env update --file "${this_dir}/environment.yml" --prune
3535

36-
# 4. Build codecs
37-
build_tools/setup_helpers/build_third_party.sh
36+
# 4. Buld codecs
37+
mkdir -p third_party/build
38+
(
39+
cd third_party/build
40+
cmake ..
41+
cmake --build .
42+
)

‎.gitignore

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,3 +121,9 @@ gen.yml
121121
examples/interactive_asr/data/*.txt
122122
examples/interactive_asr/data/*.model
123123
examples/interactive_asr/data/*.pt
124+
125+
# third parties
126+
third_party/archives/
127+
third_party/install/
128+
third_party/src/
129+
third_party/tmp/

‎README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -113,8 +113,8 @@ python setup.py install
113113
MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ python setup.py install
114114
```
115115

116-
Alternatively, the build process can build SoX (and codecs such as libmad, lame and flac) statically and torchaudio can link them, by setting environment variable `BUILD_SOX=1`.
117-
The build process will fetch and build SoX, liblame, libmad, flac before building extension.
116+
Alternatively, the build process can build libsox and some optional codecs statically and torchaudio can link them, by setting environment variable `BUILD_SOX=1`.
117+
The build process will fetch and build libmad, lame, flac, vorbis, opus, and libsox before building extension. This process requires `cmake` and `pkg-config`.
118118

119119
```bash
120120
# Linux

‎build_tools/setup_helpers/build_third_party.sh

Lines changed: 0 additions & 64 deletions
This file was deleted.

0 commit comments

Comments
 (0)