Bug 1314147 - Port the libvpx mozbuild generator to aom. r=froydnj,kinetik

This is a port of the libvpx scripts, themselves a port of
Chromium's scripts to generate an external build description
using hooks in the upstream configure and make scripts.
The libaom library is a fork of libvpx so we can use a similar
approach.

I've put the upstream source in $(topsrc_dir)/third_party/aom
but the build description and any patches are under the media
directory with the other codecs, similar to how zlib works.

Config files and headers generated by the upstream build system
are also under $(topsrc_dir)/media/libaom/.

MozReview-Commit-ID: ATLgOTPD2i1

--HG--
extra : rebase_source : c0fdcdb41a5bac5fdf64f773458cb62937fc9dd8
This commit is contained in:
Ralph Giles 2017-04-12 17:01:43 -07:00
parent 1b95866069
commit a3714d238c
4 changed files with 524 additions and 0 deletions

View File

@ -0,0 +1,19 @@
This directory contains build files for the aom video
codec reference implementation. The actual library
source is in $TOPSRCDIR/third_party/aom/
Any patches or additional configuration to be applied to the
upstream source should be kept here in the media/libaom
directory.
To update the library source and build config files, execute
./mach vendor aom
To update to a specific upstream git tag or commit, use
./mach vendor aom -r <commit>
The upstream aom git repository is:
https://aomedia.googlesource.com/aom

View File

@ -0,0 +1,270 @@
#!/bin/bash -e
#
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# Modified from chromium/src/third_party/libaom/generate_gni.sh
# This script is used to generate sources.mozbuild and files in the
# config/platform directories needed to build libaom.
# Every time libaom source code is updated just run this script.
#
# Usage:
# $ ./generate_sources_mozbuild.sh
export LC_ALL=C
BASE_DIR=$(pwd)
LIBAOM_SRC_DIR="../../third_party/aom"
LIBAOM_CONFIG_DIR="config"
# Print license header.
# $1 - Output base name
function write_license {
echo "# This file is generated. Do not edit." >> $1
echo "" >> $1
}
# Search for source files with the same basename.
# The build does not support duplicate file names.
function find_duplicates {
local readonly duplicate_file_names=$(find \
$BASE_DIR/$LIBAOM_SRC_DIR/vp8 \
$BASE_DIR/$LIBAOM_SRC_DIR/vp9 \
$BASE_DIR/$LIBAOM_SRC_DIR/av1 \
$BASE_DIR/$LIBAOM_SRC_DIR/aom \
$BASE_DIR/$LIBAOM_SRC_DIR/aom_dsp \
-type f -name \*.c | xargs -I {} basename {} | sort | uniq -d \
)
if [ -n "${duplicate_file_names}" ]; then
echo "ERROR: DUPLICATE FILES FOUND"
for file in ${duplicate_file_names}; do
find \
$BASE_DIR/$LIBAOM_SRC_DIR/vp8 \
$BASE_DIR/$LIBAOM_SRC_DIR/vp9 \
$BASE_DIR/$LIBAOM_SRC_DIR/av1 \
$BASE_DIR/$LIBAOM_SRC_DIR/aom \
$BASE_DIR/$LIBAOM_SRC_DIR/aom_dsp \
-name $file
done
exit 1
fi
}
# Generate sources.mozbuild with a list of source files.
# $1 - Array name for file list. This is processed with 'declare' below to
# regenerate the array locally.
# $2 - Variable name.
# $3 - Output file.
function write_sources {
# Convert the first argument back in to an array.
declare -a file_list=("${!1}")
echo " '$2': [" >> "$3"
for f in $file_list
do
echo " '$LIBAOM_SRC_DIR/$f'," >> "$3"
done
echo "]," >> "$3"
}
# Convert a list of source files into sources.mozbuild.
# $1 - Input file.
# $2 - Output prefix.
function convert_srcs_to_project_files {
# Do the following here:
# 1. Filter .c, .h, .s, .S and .asm files.
# 3. Convert .asm.s to .asm because moz.build will do the conversion.
local source_list=$(grep -E '(\.c|\.h|\.S|\.s|\.asm)$' $1)
# Remove aom_config.c.
source_list=$(echo "$source_list" | grep -v 'aom_config\.c')
# The actual ARM files end in .asm. We have rules to translate them to .S
source_list=$(echo "$source_list" | sed s/\.asm\.s$/.asm/)
# Exports - everything in aom, aom_mem, aom_ports, aom_scale
local exports_list=$(echo "$source_list" | \
egrep '^(aom|aom_mem|aom_ports|aom_scale)/.*h$')
# but not anything in one level down, like 'internal'
exports_list=$(echo "$exports_list" | egrep -v '/(internal|src)/')
# or any of the other internal-ish header files.
exports_list=$(echo "$exports_list" | egrep -v '/(emmintrin_compat.h|mem_.*|msvc.h|aom_once.h)$')
# Remove these files from the main list.
source_list=$(comm -23 <(echo "$source_list") <(echo "$exports_list"))
# Write a single file that includes all source files for all archs.
local c_sources=$(echo "$source_list" | egrep '.(asm|c)$')
local exports_sources=$(echo "$exports_list" | egrep '.h$')
write_sources exports_sources ${2}_EXPORTS "$BASE_DIR/sources.mozbuild"
write_sources c_sources ${2}_SOURCES "$BASE_DIR/sources.mozbuild"
}
# Clean files from previous make.
function make_clean {
make clean > /dev/null
rm -f libaom_srcs.txt
}
# Print the configuration.
# $1 - Header file directory.
function print_config {
$BASE_DIR/lint_config.sh -p \
-h $BASE_DIR/$LIBAOM_CONFIG_DIR/$1/aom_config.h \
-a $BASE_DIR/$LIBAOM_CONFIG_DIR/$1/aom_config.asm
}
# Generate *_rtcd.h files.
# $1 - Header file directory.
# $2 - Architecture.
# $3 - Optional - any additional arguments to pass through.
function gen_rtcd_header {
echo "Generate $LIBAOM_CONFIG_DIR/$1/*_rtcd.h files."
rm -rf $TEMP_DIR/libaom.config
$BASE_DIR/lint_config.sh -p \
-h $BASE_DIR/$LIBAOM_CONFIG_DIR/$1/aom_config.h \
-a $BASE_DIR/$LIBAOM_CONFIG_DIR/$1/aom_config.asm \
-o $TEMP_DIR/libaom.config
$BASE_DIR/$LIBAOM_SRC_DIR/build/make/rtcd.pl \
--arch=$2 \
--sym=aom_rtcd $DISABLE_AVX $3 \
--config=$TEMP_DIR/libaom.config \
$BASE_DIR/$LIBAOM_SRC_DIR/av1/common/av1_rtcd_defs.pl \
> $BASE_DIR/$LIBAOM_CONFIG_DIR/$1/av1_rtcd.h
$BASE_DIR/$LIBAOM_SRC_DIR/build/make/rtcd.pl \
--arch=$2 \
--sym=aom_scale_rtcd $DISABLE_AVX $3 \
--config=$TEMP_DIR/libaom.config \
$BASE_DIR/$LIBAOM_SRC_DIR/aom_scale/aom_scale_rtcd.pl \
> $BASE_DIR/$LIBAOM_CONFIG_DIR/$1/aom_scale_rtcd.h
$BASE_DIR/$LIBAOM_SRC_DIR/build/make/rtcd.pl \
--arch=$2 \
--sym=aom_dsp_rtcd $DISABLE_AVX $3 \
--config=$TEMP_DIR/libaom.config \
$BASE_DIR/$LIBAOM_SRC_DIR/aom_dsp/aom_dsp_rtcd_defs.pl \
> $BASE_DIR/$LIBAOM_CONFIG_DIR/$1/aom_dsp_rtcd.h
rm -rf $TEMP_DIR/libaom.config
}
# Generate Config files. "--enable-external-build" must be set to skip
# detection of capabilities on specific targets.
# $1 - Header file directory.
# $2 - Config command line.
function gen_config_files {
./configure $2 #> /dev/null
# Disable HAVE_UNISTD_H.
( echo '/HAVE_UNISTD_H'; echo 'd' ; echo 'w' ; echo 'q' ) | ed -s aom_config.h
local ASM_CONV=ads2gas.pl
# Generate aom_config.asm.
if [[ "$1" == *x64* ]] || [[ "$1" == *ia32* ]]; then
egrep "#define [A-Z0-9_]+ [01]" aom_config.h | awk '{print "%define " $2 " " $3}' > aom_config.asm
else
egrep "#define [A-Z0-9_]+ [01]" aom_config.h | awk '{print $2 " EQU " $3}' | perl $BASE_DIR/$LIBAOM_SRC_DIR/build/make/$ASM_CONV > aom_config.asm
fi
mkdir -p $BASE_DIR/$LIBAOM_CONFIG_DIR/$1
cp aom_config.* $BASE_DIR/$LIBAOM_CONFIG_DIR/$1
make_clean
rm -rf aom_config.*
}
find_duplicates
echo "Creating temporary directory."
TEMP_DIR="$BASE_DIR/.temp"
rm -rf $TEMP_DIR
cp -R $LIBAOM_SRC_DIR $TEMP_DIR
cd $TEMP_DIR
echo "Generating config files."
all_platforms="--enable-external-build --disable-examples --disable-docs --disable-unit-tests"
all_platforms="${all_platforms} --size-limit=8192x4608 --enable-pic"
x86_platforms="--enable-postproc --as=yasm"
arm_platforms="--enable-runtime-cpu-detect --enable-realtime-only"
gen_config_files linux/x64 "--target=x86_64-linux-gcc ${all_platforms} ${x86_platforms}"
gen_config_files linux/ia32 "--target=x86-linux-gcc ${all_platforms} ${x86_platforms}"
gen_config_files mac/x64 "--target=x86_64-darwin9-gcc ${all_platforms} ${x86_platforms}"
gen_config_files win/x64 "--target=x86_64-win64-vs12 ${all_platforms} ${x86_platforms}"
gen_config_files win/ia32 "--target=x86-win32-gcc ${all_platforms} ${x86_platforms}"
gen_config_files linux/arm "--target=armv7-linux-gcc ${all_platforms} ${arm_platforms}"
gen_config_files generic "--target=generic-gnu ${all_platforms}"
echo "Remove temporary directory."
cd $BASE_DIR
rm -rf $TEMP_DIR
echo "Create temporary directory."
TEMP_DIR="$BASE_DIR/.temp"
rm -rf $TEMP_DIR
cp -R $LIBAOM_SRC_DIR $TEMP_DIR
cd $TEMP_DIR
gen_rtcd_header linux/x64 x86_64
gen_rtcd_header linux/ia32 x86
gen_rtcd_header mac/x64 x86_64
gen_rtcd_header win/x64 x86_64
gen_rtcd_header win/ia32 x86
gen_rtcd_header linux/arm armv7
gen_rtcd_header generic generic
echo "Prepare Makefile."
./configure --target=generic-gnu > /dev/null
make_clean
# Remove existing source files.
rm -rf $BASE_DIR/sources.mozbuild
write_license $BASE_DIR/sources.mozbuild
echo "files = {" >> $BASE_DIR/sources.mozbuild
echo "Generate X86_64 source list."
config=$(print_config linux/x64)
make_clean
make libaom_srcs.txt target=libs $config > /dev/null
convert_srcs_to_project_files libaom_srcs.txt X64
# Copy aom_version.h once. The file is the same for all platforms.
cp aom_version.h $BASE_DIR/$LIBAOM_CONFIG_DIR
echo "Generate IA32 source list."
config=$(print_config linux/ia32)
make_clean
make libaom_srcs.txt target=libs $config > /dev/null
convert_srcs_to_project_files libaom_srcs.txt IA32
echo "Generate ARM source list."
config=$(print_config linux/arm)
make_clean
make libaom_srcs.txt target=libs $config > /dev/null
convert_srcs_to_project_files libaom_srcs.txt ARM
echo "Generate generic source list."
config=$(print_config generic)
make_clean
make libaom_srcs.txt target=libs $config > /dev/null
convert_srcs_to_project_files libaom_srcs.txt GENERIC
echo "}" >> $BASE_DIR/sources.mozbuild
echo "Remove temporary directory."
cd $BASE_DIR
rm -rf $TEMP_DIR
cd $BASE_DIR/$LIBAOM_SRC_DIR
cd $BASE_DIR

112
media/libaom/lint_config.sh Executable file
View File

@ -0,0 +1,112 @@
#!/bin/bash -e
#
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# This script is used to compare vpx_config.h and vpx_config.asm to
# verify the two files match.
#
# Arguments:
#
# -h - C Header file.
# -a - ASM file.
# -p - Print the options if correct.
# -o - Output file.
#
# Usage:
#
# # Compare the two configuration files and output the final results.
# ./lint_config.sh -h vpx_config.h -a vpx_config.asm -o libvpx.config -p
export LC_ALL=C
print_final="no"
while getopts "h:a:o:p" flag
do
if [ "$flag" = "h" ]; then
header_file=$OPTARG
elif [ "$flag" = "a" ]; then
asm_file=$OPTARG
elif [ "$flag" = "o" ]; then
out_file=$OPTARG
elif [ "$flag" = "p" ]; then
print_final="yes"
fi
done
if [ -z "$header_file" ]; then
echo "Header file not specified."
false
exit
fi
if [ -z "$asm_file" ]; then
echo "ASM file not specified."
false
exit
fi
# Concat header file and assembly file and select those ended with 0 or 1.
combined_config="$(cat $header_file $asm_file | grep -E ' +[01] *$')"
# Extra filtering for known exceptions.
combined_config="$(echo "$combined_config" | grep -v WIDE_REFERENCE)"
combined_config="$(echo "$combined_config" | grep -v ARCHITECTURE)"
combined_config="$(echo "$combined_config" | grep -v DO1STROUNDING)"
# Remove all spaces.
combined_config="$(echo "$combined_config" | sed 's/[ \t]//g')"
# Remove #define in the header file.
combined_config="$(echo "$combined_config" | sed 's/.*define//')"
# Remove equ in the ASM file.
combined_config="$(echo "$combined_config" | sed 's/\.equ//')" # gas style
combined_config="$(echo "$combined_config" | sed 's/equ//')" # rvds style
combined_config="$(echo "$combined_config" | sed 's/\.set//')" # apple style
# Remove %define in YASM ASM files.
combined_config="$(echo "$combined_config" | sed 's/%define\s *//')" # yasm style
# Remove useless comma in gas style assembly file.
combined_config="$(echo "$combined_config" | sed 's/,//')"
# Substitute 0 with =no.
combined_config="$(echo "$combined_config" | sed 's/0$/=no/')"
# Substitute 1 with =yes.
combined_config="$(echo "$combined_config" | sed 's/1$/=yes/')"
# Find the mismatch variables.
odd_config="$(echo "$combined_config" | sort | uniq -u)"
odd_vars="$(echo "$odd_config" | sed 's/=.*//' | uniq)"
for var in $odd_vars; do
echo "Error: Configuration mismatch for $var."
echo "Header file: $header_file"
echo "$(cat -n $header_file | grep "$var[ \t]")"
echo "Assembly file: $asm_file"
echo "$(cat -n $asm_file | grep "$var[ \t]")"
echo ""
done
if [ -n "$odd_vars" ]; then
false
exit
fi
if [ "$print_final" = "no" ]; then
exit
fi
# Do some additional filter to make libvpx happy.
combined_config="$(echo "$combined_config" | grep -v ARCH_X86=no)"
combined_config="$(echo "$combined_config" | grep -v ARCH_X86_64=no)"
# Print out the unique configurations.
if [ -n "$out_file" ]; then
echo "$combined_config" | sort | uniq > $out_file
else
echo "$combined_config" | sort | uniq
fi

123
media/libaom/moz.build Normal file
View File

@ -0,0 +1,123 @@
# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
# vim: set filetype=python:
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
with Files('*'):
BUG_COMPONENT = ('Core', 'Audio/Video')
include('sources.mozbuild')
# Linux, Mac and Win share file lists for x86* but not configurations.
if CONFIG['CPU_ARCH'] == 'x86_64':
EXPORTS.aom += files['X64_EXPORTS']
SOURCES += files['X64_SOURCES']
if CONFIG['OS_TARGET'] == 'WINNT':
ASFLAGS += [ '-I%s/media/libaom/config/win/x64/' % TOPSRCDIR ]
CFLAGS += [ '-I%s/media/libaom/config/win/x64/' % TOPSRCDIR ]
elif CONFIG['OS_TARGET'] == 'Darwin':
ASFLAGS += [ '-I%s/media/libaom/config/mac/x64/' % TOPSRCDIR ]
CFLAGS += [ '-I%s/media/libaom/config/mac/x64/' % TOPSRCDIR ]
else: # Android, Linux, BSDs, etc.
ASFLAGS += [ '-I%s/media/libaom/config/linux/x64/' % TOPSRCDIR ]
CFLAGS += [ '-I%s/media/libaom/config/linux/x64/' % TOPSRCDIR ]
elif CONFIG['CPU_ARCH'] == 'x86':
EXPORTS.aom += files['IA32_EXPORTS']
SOURCES += files['IA32_SOURCES']
if CONFIG['OS_TARGET'] == 'WINNT':
ASFLAGS += [ '-I%s/media/libaom/config/win/ia32/' % TOPSRCDIR ]
CFLAGS += [ '-I%s/media/libaom/config/win/ia32/' % TOPSRCDIR ]
elif CONFIG['OS_TARGET'] == 'Darwin':
ASFLAGS += [ '-I%s/media/libaom/config/mac/ia32/' % TOPSRCDIR ]
CFLAGS += [ '-I%s/media/libaom/config/mac/ia32/' % TOPSRCDIR ]
else: # Android, Linux, BSDs, etc.
ASFLAGS += [ '-I%s/media/libaom/config/linux/ia32/' % TOPSRCDIR ]
CFLAGS += [ '-I%s/media/libaom/config/linux/ia32/' % TOPSRCDIR ]
elif CONFIG['CPU_ARCH'] == 'arm':
EXPORTS.aom += files['ARM_EXPORTS']
ASFLAGS += [
'-I%s/media/libaom/config/linux/arm/' % TOPSRCDIR,
'-I%s/libaom' % OBJDIR,
]
CFLAGS += [ '-I%s/media/libaom/config/linux/arm/' % TOPSRCDIR ]
arm_asm_files = files['ARM_SOURCES']
if CONFIG['VPX_AS_CONVERSION']:
SOURCES += sorted([
"!%s.S" % f if f.endswith('.asm') else f for f in arm_asm_files
])
else:
SOURCES += sorted(arm_asm_files)
for f in SOURCES:
if f.endswith('neon.c'):
SOURCES[f].flags += CONFIG['VPX_ASFLAGS']
if CONFIG['OS_TARGET'] == 'Android':
# For cpu-features.h
LOCAL_INCLUDES += [
'%%%s/sources/android/cpufeatures' % CONFIG['ANDROID_NDK'],
]
if CONFIG['CLANG_CXX']:
ASFLAGS += [
'-no-integrated-as',
]
else:
# Generic C-only configuration
EXPORTS.aom += files['GENERIC_EXPORTS']
SOURCES += files['GENERIC_SOURCES']
ASFLAGS += [ '-I%s/media/libaom/config/generic/' % TOPSRCDIR ]
CFLAGS += [ '-I%s/media/libaom/config/generic/' % TOPSRCDIR ]
# We allow warnings for third-party code that can be updated from upstream.
ALLOW_COMPILER_WARNINGS = True
FINAL_LIBRARY = 'gkmedias'
if CONFIG['OS_TARGET'] == 'Android':
# Older versions of the Android NDK don't pre-define anything to indicate
# the OS they're on, so do it for them.
DEFINES['__linux__'] = True
if not CONFIG['MOZ_WEBRTC']:
SOURCES += [
'%%%s/sources/android/cpufeatures/cpu-features.c' % CONFIG['ANDROID_NDK'],
]
if CONFIG['CLANG_CL'] or not CONFIG['_MSC_VER']:
for f in SOURCES:
if f.endswith('sse2.c'):
SOURCES[f].flags += CONFIG['SSE2_FLAGS']
elif f.endswith('ssse3.c'):
SOURCES[f].flags += ['-mssse3']
elif f.endswith('sse4.c'):
SOURCES[f].flags += ['-msse4.1']
elif f.endswith('avx.c'):
SOURCES[f].flags += ['-mavx']
elif f.endswith('avx2.c'):
SOURCES[f].flags += ['-mavx2']
# Suppress warnings in third-party code.
if CONFIG['GNU_CC'] or CONFIG['CLANG_CL']:
CFLAGS += [
'-Wno-sign-compare',
'-Wno-unused-function', # so many of these warnings; just ignore them
]
if CONFIG['CLANG_CXX'] or CONFIG['CLANG_CL']:
CFLAGS += [
'-Wno-unreachable-code',
'-Wno-unneeded-internal-declaration',
]
ASFLAGS += CONFIG['VPX_ASFLAGS']
ASFLAGS += [
'-I.',
'-I%s/third_party/aom' % TOPSRCDIR,
]
CFLAGS += [
'-I%s/third_party/aom' % TOPSRCDIR,
'-I%s/media/libaom/config' % TOPSRCDIR, # aom_version.h
]