252 lines
9.4 KiB
Python
252 lines
9.4 KiB
Python
# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
"""
|
|
This file is used in CI generate binary files for different kinds of apps
|
|
"""
|
|
|
|
import argparse
|
|
import sys
|
|
import os
|
|
import re
|
|
import logging
|
|
from pathlib import Path
|
|
from typing import List
|
|
import typing as t
|
|
import subprocess
|
|
import shutil
|
|
|
|
from idf_build_apps import App, build_apps, find_apps, setup_logging
|
|
from idf_build_apps.app import CMakeApp
|
|
|
|
logger = logging.getLogger('idf_build_apps')
|
|
IDF_PATH = os.getenv("IDF_PATH", "")
|
|
|
|
PROJECT_ROOT = Path(__file__).parent.parent.parent.absolute()
|
|
APPS_BUILD_PER_JOB = 30
|
|
IGNORE_WARNINGS = [
|
|
r"warning: unused variable 'head'",
|
|
r"WARNING: The following Kconfig variables were used in",
|
|
r"unknown kconfig symbol",
|
|
r"warning: assignment discards 'const' qualifier from pointer target type", # For Speaker
|
|
r"'esp_lcd_touch_get_coordinates' is deprecated: This API will be removed in version 2.0.0. Use esp_lcd_touch_get_data instead!", # For Speaker
|
|
]
|
|
|
|
class CustomApp(CMakeApp):
|
|
build_system: t.Literal['custom'] = 'custom' # Must be unique to identify your custom app type
|
|
|
|
def _build(
|
|
self,
|
|
*,
|
|
manifest_rootpath: t.Optional[str] = None,
|
|
modified_components: t.Optional[t.List[str]] = None,
|
|
modified_files: t.Optional[t.List[str]] = None,
|
|
check_app_dependencies: bool = False,
|
|
) -> None:
|
|
self._pre_hook()
|
|
super()._build(
|
|
manifest_rootpath=manifest_rootpath,
|
|
modified_components=modified_components,
|
|
modified_files=modified_files,
|
|
check_app_dependencies=check_app_dependencies,
|
|
)
|
|
|
|
def is_board_manager_project(self) -> bool:
|
|
main_yml_path = Path(self.work_dir)/"main"/"idf_component.yml"
|
|
with open(main_yml_path) as f:
|
|
return 'esp_board_manager' in f.read()
|
|
|
|
def get_board_name_from_sdkconfig(self, sdkconfig_path: str) -> str:
|
|
board_name = ''
|
|
sdkconfig_path = Path(sdkconfig_path).name
|
|
if sdkconfig_path.startswith('sdkconfig.ci.board.'):
|
|
board_name = sdkconfig_path[len('sdkconfig.ci.board.'):]
|
|
return board_name
|
|
|
|
def clear_project_generated_files(self):
|
|
proj_path = Path(self.work_dir).absolute()
|
|
# Remove directories
|
|
shutil.rmtree(proj_path/"managed_components", ignore_errors=True)
|
|
shutil.rmtree(proj_path/"components"/"gen_bmgr_codes", ignore_errors=True)
|
|
shutil.rmtree(proj_path/"build", ignore_errors=True)
|
|
# Remove files
|
|
(proj_path/"dependencies.lock").unlink(missing_ok=True)
|
|
(proj_path/"sdkconfig").unlink(missing_ok=True)
|
|
(proj_path/"board_manager.defaults").unlink(missing_ok=True)
|
|
|
|
def _pre_hook(self):
|
|
board_name = ''
|
|
if self.is_board_manager_project() and len(self.sdkconfig_files) > 1:
|
|
board_name = self.get_board_name_from_sdkconfig(self.sdkconfig_files[1])
|
|
print(f'== Pre build hook for app: {self.name} at \'{self.work_dir}\' for target: {self.target}, board: {board_name}, argv: {sys.argv}, sdkconfig_files: {self.sdkconfig_files}')
|
|
self.clear_project_generated_files()
|
|
if board_name == '':
|
|
print(f'== No board name found, skip setting board manager config')
|
|
return # no board name found, skip the pre build hook
|
|
subprocess.run([sys.executable, f"{IDF_PATH}/tools/idf.py", "gen-bmgr-config", "-c", str(Path(self.work_dir).absolute()/"boards"), "-b", board_name], cwd=self.work_dir)
|
|
subprocess.run([sys.executable, f"{IDF_PATH}/tools/idf.py", "set-target", self.target], cwd=self.work_dir)
|
|
|
|
def _get_idf_version():
|
|
if os.environ.get('IDF_VERSION'):
|
|
return os.environ.get('IDF_VERSION')
|
|
version_path = os.path.join(os.environ['IDF_PATH'], 'tools/cmake/version.cmake')
|
|
regex = re.compile(r'^\s*set\s*\(\s*IDF_VERSION_([A-Z]{5})\s+(\d+)')
|
|
ver = {}
|
|
with open(version_path) as f:
|
|
for line in f:
|
|
m = regex.match(line)
|
|
if m:
|
|
ver[m.group(1)] = m.group(2)
|
|
return '{}.{}'.format(int(ver['MAJOR']), int(ver['MINOR']))
|
|
|
|
def get_cmake_apps(
|
|
paths,
|
|
target,
|
|
config_rules_str,
|
|
default_build_targets,
|
|
): # type: (List[str], str, List[str]) -> List[App]
|
|
idf_ver = _get_idf_version()
|
|
apps = find_apps(
|
|
paths,
|
|
recursive=True,
|
|
target=target,
|
|
build_dir=f'{idf_ver}/build_@t_@w',
|
|
config_rules_str=config_rules_str,
|
|
build_log_filename='build_log.txt',
|
|
size_json_filename='size.json',
|
|
check_warnings=True,
|
|
no_preserve=False,
|
|
default_build_targets=default_build_targets,
|
|
manifest_files=[
|
|
str(Path(PROJECT_ROOT)/'.build-rules.yml'),
|
|
],
|
|
build_system=CustomApp,
|
|
)
|
|
return apps
|
|
|
|
|
|
def main(args): # type: (argparse.Namespace) -> None
|
|
default_build_targets = args.default_build_targets.split(',') if args.default_build_targets else None
|
|
# Handle default config values if not provided
|
|
if args.config:
|
|
# Support both semicolon-separated strings and multiple --config arguments
|
|
config_list = []
|
|
for config_item in args.config:
|
|
# Split by semicolon if present, otherwise use as-is
|
|
if ';' in config_item:
|
|
config_list.extend([item.strip() for item in config_item.split(';') if item.strip()])
|
|
else:
|
|
config_list.append(config_item)
|
|
else:
|
|
config_list = ['sdkconfig.defaults=defaults', 'sdkconfig.ci.*=', '=defaults']
|
|
apps = get_cmake_apps(args.paths, args.target, config_list, default_build_targets)
|
|
|
|
if args.find:
|
|
if args.output:
|
|
os.makedirs(os.path.dirname(os.path.realpath(args.output)), exist_ok=True)
|
|
with open(args.output, 'w') as fw:
|
|
for app in apps:
|
|
fw.write(app.to_json() + '\n')
|
|
else:
|
|
for app in apps:
|
|
print(app)
|
|
|
|
sys.exit(0)
|
|
|
|
if args.exclude_apps:
|
|
apps_to_build = [app for app in apps if app.name not in args.exclude_apps]
|
|
else:
|
|
apps_to_build = apps[:]
|
|
|
|
logger.info('Found %d apps after filtering', len(apps_to_build))
|
|
logger.info(
|
|
'Suggest setting the parallel count to %d for this build job',
|
|
len(apps_to_build) // APPS_BUILD_PER_JOB + 1,
|
|
)
|
|
|
|
ret_code = build_apps(
|
|
apps_to_build,
|
|
parallel_count=args.parallel_count,
|
|
parallel_index=args.parallel_index,
|
|
dry_run=False,
|
|
collect_size_info=args.collect_size_info,
|
|
keep_going=True,
|
|
ignore_warning_strs=IGNORE_WARNINGS,
|
|
copy_sdkconfig=True,
|
|
no_preserve=False,
|
|
)
|
|
|
|
sys.exit(ret_code)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser(
|
|
description='Build all the apps for different test types. Will auto remove those non-test apps binaries',
|
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
)
|
|
parser.add_argument('paths', nargs='*', help='Paths to the apps to build.')
|
|
parser.add_argument(
|
|
'-t', '--target',
|
|
default='all',
|
|
help='Build apps for given target. could pass "all" to get apps for all targets',
|
|
)
|
|
parser.add_argument(
|
|
'--config',
|
|
default=None,
|
|
action='append',
|
|
help='Adds configurations (sdkconfig file names) to build. This can either be '
|
|
'FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, '
|
|
'relative to the project directory, to be used. Optional NAME can be specified, '
|
|
'which can be used as a name of this configuration. FILEPATTERN is the name of '
|
|
'the sdkconfig file, relative to the project directory, with at most one wildcard. '
|
|
'The part captured by the wildcard is used as the name of the configuration. '
|
|
'Can be specified multiple times to add multiple configurations, or use semicolon '
|
|
'to separate multiple configs in a single argument: --config="config1;config2;config3". '
|
|
'If not specified, defaults to: sdkconfig.defaults=defaults, sdkconfig.ci.*=, =defaults. '
|
|
)
|
|
parser.add_argument(
|
|
'--parallel-count', default=1, type=int, help='Number of parallel build jobs.'
|
|
)
|
|
parser.add_argument(
|
|
'--parallel-index',
|
|
default=1,
|
|
type=int,
|
|
help='Index (1-based) of the job, out of the number specified by --parallel-count.',
|
|
)
|
|
parser.add_argument(
|
|
'--collect-size-info',
|
|
type=argparse.FileType('w'),
|
|
help='If specified, the test case name and size info json will be written to this file',
|
|
)
|
|
parser.add_argument(
|
|
'--exclude-apps',
|
|
nargs='*',
|
|
help='Exclude build apps',
|
|
)
|
|
parser.add_argument(
|
|
'--default-build-targets',
|
|
default=None,
|
|
help='default build targets used in manifest files',
|
|
)
|
|
parser.add_argument(
|
|
'-v', '--verbose',
|
|
action='count', default=0,
|
|
help='Show verbose log message',
|
|
)
|
|
parser.add_argument(
|
|
'--find',
|
|
action='store_true',
|
|
help='Find the buildable applications. If enable this option, build options will be ignored.',
|
|
)
|
|
parser.add_argument(
|
|
'-o', '--output',
|
|
help='Print the found apps to the specified file instead of stdout'
|
|
)
|
|
|
|
arguments = parser.parse_args()
|
|
if not arguments.paths:
|
|
arguments.paths = [PROJECT_ROOT]
|
|
setup_logging(verbose=arguments.verbose) # Info
|
|
main(arguments)
|