From 435587484b8bf0a68e9935bff0692c10a86dc19e Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Sun, 12 Jul 2015 18:02:30 +0800 Subject: [PATCH 01/14] Update parse_options to use argparse instead of the deprecated optparse. Fix 'extra_options' so that meaningful and complete command line options can be added on invocation. --- fabricate.py | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/fabricate.py b/fabricate.py index 28025fd..9d2a3ad 100755 --- a/fabricate.py +++ b/fabricate.py @@ -29,7 +29,7 @@ deps_version = 2 import atexit -import optparse +import argparse import os import platform import re @@ -1455,32 +1455,40 @@ def outofdate(command): def parse_options(usage=_usage, extra_options=None, command_line=None): """ Parse command line options and return (parser, options, args). """ - parser = optparse.OptionParser(usage='Usage: %prog '+usage, - version='%prog '+__version__) - parser.disable_interspersed_args() - parser.add_option('-t', '--time', action='store_true', + parser = argparse.ArgumentParser() + parser.add_argument('--version', action='version', version='%(prog)s '+__version__) + parser.add_argument('-t', '--time', action='store_true', help='use file modification times instead of MD5 sums') - parser.add_option('-d', '--dir', action='append', + parser.add_argument('-d', '--dir', action='append', help='add DIR to list of relevant directories') - parser.add_option('-c', '--clean', action='store_true', + parser.add_argument('-c', '--clean', action='store_true', help='autoclean build outputs before running') - parser.add_option('-q', '--quiet', action='store_true', + parser.add_argument('-q', '--quiet', action='store_true', help="don't echo commands, only print errors") - parser.add_option('-D', '--debug', action='store_true', + parser.add_argument('-D', '--debug', action='store_true', help="show debug info (why commands are rebuilt)") - parser.add_option('-k', '--keep', action='store_true', + parser.add_argument('-k', '--keep', action='store_true', help='keep temporary strace output files') - parser.add_option('-j', '--jobs', type='int', + parser.add_argument('-j', '--jobs', type=int, help='maximum number of parallel jobs') + parser.add_argument('actions', nargs=argparse.REMAINDER, + help="The build actions to perform, such as 'clean', 'debug', 'release'") + if extra_options: + print(extra_options) # add any user-specified options passed in via main() for option in extra_options: - parser.add_option(option) + name_or_flags = option['name_or_flags'] + del option['name_or_flags'] + parser.add_argument(*name_or_flags,**option) + if command_line is not None: - options, args = parser.parse_args(command_line) + options = parser.parse_args(command_line) else: - options, args = parser.parse_args() - _parsed_options = (parser, options, args) + options = parser.parse_args() + + # Returns (parser, options, actions) + _parsed_options = (parser, options, options.actions) return _parsed_options def fabricate_version(min=None, max=None): @@ -1515,7 +1523,7 @@ def main(globals_dict=None, build_dir=None, extra_options=None, builder=None, and modify the command line passed to the build script. "default" is the default user script function to call, None = 'build' "extra_options" is an optional list of options created with - optparse.make_option(). The pseudo-global variable main.options + argparse.add_argument(). The pseudo-global variable main.options is set to the parsed options list. "kwargs" is any other keyword arguments to pass to the builder """ global default_builder, default_command, _pool From 6232b19a95b183900a789a48da74c6d94c7224a9 Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Sun, 12 Jul 2015 18:32:26 +0800 Subject: [PATCH 02/14] Update fabricate.py Remove extraneous test print statement --- fabricate.py | 1 - 1 file changed, 1 deletion(-) diff --git a/fabricate.py b/fabricate.py index 9d2a3ad..a523cee 100755 --- a/fabricate.py +++ b/fabricate.py @@ -1475,7 +1475,6 @@ def parse_options(usage=_usage, extra_options=None, command_line=None): help="The build actions to perform, such as 'clean', 'debug', 'release'") if extra_options: - print(extra_options) # add any user-specified options passed in via main() for option in extra_options: name_or_flags = option['name_or_flags'] From 0443679dc1b09abea3c972aa63b9b6b5d2d17c83 Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Thu, 24 Sep 2015 14:31:55 +0800 Subject: [PATCH 03/14] Add __init__ so we can use directly as a module --- __init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 __init__.py diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 From 0e52b5ed8b29ab9bd00a86411b87d757d34492e4 Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Thu, 7 Jun 2018 10:20:49 +0800 Subject: [PATCH 04/14] cleanup python3/2 mess only support python 3.4+ base runner class missing _builder. add init for it and propogate changes to subclasses to call base class init. refix args after I broke it during upstream merge. --- fabricate.py | 60 +++++++++++++++++++++++----------------------------- 1 file changed, 27 insertions(+), 33 deletions(-) diff --git a/fabricate.py b/fabricate.py index fcd3413..d3e3583 100755 --- a/fabricate.py +++ b/fabricate.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Build tool that finds dependencies automatically for any language. @@ -18,12 +18,14 @@ from fabricate import * help(function) +This version ONLY runs on Python 3. + """ -from __future__ import with_statement, print_function, unicode_literals +# from __future__ import with_statement, print_function, unicode_literals # fabricate version number -__version__ = '1.29.3' +__version__ = '3.00.1' # if version of .deps file has changed, we know to not use it deps_version = 2 @@ -41,27 +43,16 @@ import time import threading # NB uses old camelCase names for backward compatibility import traceback -# multiprocessing module only exists on Python >= 2.6 -try: - import multiprocessing -except ImportError: - class MultiprocessingModule(object): - def __getattr__(self, name): - raise NotImplementedError("multiprocessing module not available, can't do parallel builds") - multiprocessing = MultiprocessingModule() +import multiprocessing -# compatibility -PY3 = sys.version_info[0] == 3 -if PY3: - string_types = str - threading_condition = threading.Condition -else: - string_types = basestring +PY_MAJOR_REQUIRED = 3 +PY_MINOR_REQUIRED = 4 -try: - threading_condition = threading._Condition -except ImportError: - threading_condition = threading.Condition +# compatibility +if (sys.version_info.major != PY_MAJOR_REQUIRED) or (sys.version_info.minor < PY_MINOR_REQUIRED): + sys.stderr.write(("fabricate REQUIRES Python %d.%d. Your version = %s ") % + (PY_MAJOR_REQUIRED, PY_MINOR_REQUIRED, sys.version)) + sys.exit() # so you can do "from fabricate import *" to simplify your build script __all__ = ['setup', 'run', 'autoclean', 'main', 'shell', 'fabricate_version', @@ -135,7 +126,7 @@ def args_to_list(args): if isinstance(arg, (list, tuple)): arglist.extend(args_to_list(arg)) else: - if not isinstance(arg, string_types): + if not isinstance(arg, str): arg = str(arg) arglist.append(arg) return arglist @@ -248,6 +239,9 @@ class RunnerUnsupportedException(Exception): pass class Runner(object): + def __init__(self, builder): + self._builder = builder + def __call__(self, *args, **kwargs): """ Run command and return (dependencies, outputs), where dependencies is a list of the filenames of files that the @@ -265,7 +259,7 @@ def ignore(self, name): class AtimesRunner(Runner): def __init__(self, builder): - self._builder = builder + super().__init__(builder) self.atimes = AtimesRunner.has_atimes(self._builder.dirs) if self.atimes == 0: raise RunnerUnsupportedException( @@ -504,10 +498,10 @@ class StraceRunner(Runner): keep_temps = False def __init__(self, builder, build_dir=None): + super().__init__(builder) self.strace_system_calls = StraceRunner.get_strace_system_calls() if self.strace_system_calls is None: raise RunnerUnsupportedException('strace is not available') - self._builder = builder self.temp_count = 0 self.build_dir = os.path.abspath(build_dir or os.getcwd()) @@ -752,7 +746,7 @@ def __call__(self, *args, **kwargs): class AlwaysRunner(Runner): def __init__(self, builder): - pass + super().__init__(builder) def __call__(self, *args, **kwargs): """ Runner that always runs given command, used as a backup in case @@ -766,7 +760,7 @@ class SmartRunner(Runner): """ Smart command runner that uses StraceRunner if it can, otherwise AtimesRunner if available, otherwise AlwaysRunner. """ def __init__(self, builder): - self._builder = builder + super().__init__(builder) try: self._runner = StraceRunner(self._builder) except RunnerUnsupportedException: @@ -955,8 +949,8 @@ def _results_handler( builder, delay=0.01): r.results = False _groups.set_ok(a.do.group, False) _groups.dec_count(a.do.group) - elif isinstance(a.do, threading_condition): - # is this only for threading_condition in after()? + elif isinstance(a.do, threading.Condition): + # is this only for threading.Condition in after()? a.do.acquire() # only mark as done if there is no error a.done = no_error @@ -1196,7 +1190,7 @@ def memoize(self, command, **kwargs): This function is for compatiblity with memoize.py and is deprecated. Use run() instead. """ - if isinstance(command, string_types): + if isinstance(command, str): args = shlex.split(command) else: args = args_to_list(command) @@ -1342,7 +1336,7 @@ def set_runner(self, runner): try: self.runner = self._runner_map[runner](self) except KeyError: - if isinstance(runner, string_types): + if isinstance(runner, str): # For backwards compatibility, allow runner to be the # name of a method in a derived class: self.runner = getattr(self, runner) @@ -1492,9 +1486,9 @@ def parse_options(usage=_usage, extra_options=None, command_line=None): if command_line is not None: options = parser.parse_args(command_line) else: - options, args = parser.parse_args() + options = parser.parse_args() global _parsed_options - _parsed_options = (parser, options, args) + _parsed_options = (parser, options, options.actions) return _parsed_options def fabricate_version(min=None, max=None): From b5531d403e714ef7d85173aecfed741c1c0c70bb Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Thu, 7 Jun 2018 10:23:30 +0800 Subject: [PATCH 05/14] Add utility functions which make writing build scripts much cleaner --- generators.py | 309 ++++++++++++++++++++++++++++++++++++++++++++++++++ util.py | 242 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 551 insertions(+) create mode 100644 generators.py create mode 100644 util.py diff --git a/generators.py b/generators.py new file mode 100644 index 0000000..03757bb --- /dev/null +++ b/generators.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python3 +from fabricate import * +from util import * + +import os +import platform + +# Generic Builder for GCC and G++ +def gcc(build, section, module, tool, arch, core, src, debug=False): + gcc_opt = get_gcc_opt(build,section,module,tool,arch,core,src,debug) + gcc_opt.extend(get_gcc_opt(build,section,module,'COMMON',arch,core,src,debug)) + + # Get Includes + includes = get_includes(build,section,module,debug) + sysincludes = get_includes(build,section,module,system=True,debug=debug) + + # Add -I to each include path. + inc_opt = [['-I',inc] for inc in includes] + + # Add -i to each system include path. + sysinc_opt = [['-isystem',inc] for inc in sysincludes] + + if 'LISTING' in build[section][module]: + gcc_opt.extend(["-Wa,%s=%s" % (build[section][module]['LISTING'],get_destination_file(src,new_ext='.lst'))]) + + # GCC wont make output directories, so ensure there is a place the output can live. + output_file = get_destination_file(src,new_ext='.o') + if not os.path.isdir(os.path.dirname(output_file)): + os.makedirs(os.path.dirname(output_file)) + + tool = [join_path(build['TOOLS']['PATH'][arch],build['TOOLS'][tool][arch])] + if 'PFX' in build['TOOLS']: + if arch in build['TOOLS']['PFX']: + tool = [build['TOOLS']['PFX'][arch]] + tool + + after_modules=() + if ('USES' in build['SOURCE'][module]) : + for used_module in build['SOURCE'][module]['USES'] : + after_modules = after_modules + (used_module,) + + # run GCC + run(tool, + gcc_opt, + inc_opt,sysinc_opt, + '-c', src[0], + '-o', output_file, + group=module, + after=after_modules) + +def ar(build,section,module, debug=False): + base_dir = get_base_dir(build,section,module) + arch = build[section][module]['ARCH'] + core = build[section][module]['CORE'] + + print "Linking Library %s of %s" % ( build[section][module]['LIBRARY'], + build[section][module]['VERSION'] ) + + objects = [get_destination_file( + get_src_tuple(build,section,module,s,debug), + new_ext='.o') for s in build[section][module]['SRC']] + + library = get_destination_file( + get_src_tuple(build, + build[section][module]['LIBRARY'], + module,debug,section=section)) + + output_dir = os.path.dirname(library) + + if not os.path.isdir(output_dir): + os.makedirs(output_dir) + + # Only safe to run when all objects are build AND the directory it belongs in is made. + run(join_path(build['TOOLS']['PATH'][arch], + build['TOOLS']['AR'][arch]), + 'rcs', library, objects, + group=build[section][module]['LIBRARY'], after=(module)) + +def ld(build, section, module, debug): + base_dir = get_base_dir(build,section,module) + arch = build[section][module]['ARCH'] + core = build[section][module]['CORE'] + + print "Linking Application %s of %s" % ( build[section][module]['APP'], + build[section][module]['VERSION'] ) + + gcc_opt = get_gcc_opt(build,section,module,'GCC',arch,core,(""),debug) + + if 'LDFLAGS' in build[section][module]: + for ldflag in build[section][module]['LDFLAGS']: + gcc_opt.extend(["-Wl,"+ldflag]) + + searchpaths = [] + if 'LINK' in build[section][module]: + # Make sure we can find link include files. + searchpaths = searchpaths + [os.path.dirname(build[section][module]['LINK'])] + gcc_opt.extend(["-T,",build[section][module]['LINK']]) + + searchpaths = [["-Wl,-L,"+path] for path in searchpaths] + + for ldflag in build['OPTS']['GCC'][arch]['LDFLAGS']: + gcc_opt.extend(["-Wl,"+ldflag]) + + # Get Objects to link + objects = [get_destination_file( + get_src_tuple(build,section,module,s,debug), + new_ext='.o') for s in build[section][module]['SRC']] + + output_file = get_destination_file( + get_src_tuple(build,section,module, + build[section][module]['APP'], + debug)) + + mapfile = None + if 'MAP' in build[section][module]: + mapfile = "-Wl,-Map="+get_destination_file( + get_src_tuple(build,section,module, + build[section][module]['MAP'], + debug)) + + + # run GCC - to link + run(join_path(build['TOOLS']['PATH'][arch],build['TOOLS']['GCC'][arch]), + gcc_opt, + objects, + mapfile, + searchpaths, + '-o', output_file, + group=module+'_link', + after=(module)) + +def execute_make_script(build,section,module,debug): + cmd_cnt = 0; + + commands = build[section][module]['MAKE'] + + if debug: + if ('MAKE_DEBUG' in build[section][module]): + commands = build[section][module]['MAKE_DEBUG'] + else: + commands = [] + + for cmd in commands: + cmd_cnt += 1 + rundir = get_base_dir(build,module,section) + # Run each command in turn, preserving order on parallel builds. + run("bash", "-c", "cd "+rundir+"; "+" ".join(cmd), + group='MAKE'+module+str(cmd_cnt), + after=(module+str(cmd_cnt-1)) if cmd_cnt > 1 else None + ) + + return cmd_cnt + +def src_build(build,section,module,debug): + # Build a bunch of source files, based on their extensions. + for src in build[section][module]['SRC']: + clean_src = src + src = get_src_tuple(build,section,module,clean_src,debug) + tool = get_tool(build, src) + + if ((tool == 'GCC') or (tool == 'GXX') or (tool == 'GAS')): + gcc(build, + section, + module, + tool, + build[section][module]['ARCH'], + build[section][module]['CORE'], + src, + debug) + else: + print "%s ERROR: Don't know how to compile : %s from %s using %s" % (section, src, module, tool) + exit(1) + +def gen_hex(build,section,module,debug): + base_dir = get_base_dir(build,section,module) + arch = build[section][module]['ARCH'] + core = build[section][module]['CORE'] + + print "Making .hex of Application %s/%s" % ( build[section][module]['APP'], + build[section][module]['VERSION'] ) + + app_file = get_destination_file( + get_src_tuple(build,section,module, + build[section][module]['APP'], + debug)) + + hex_file = get_destination_file( + get_src_tuple(build,section,module, + build[section][module]['HEX'], + debug)) + + # run obj-copy - to generate .hex file + run(join_path(build['TOOLS']['PATH'][arch],build['TOOLS']['OBJ-COPY'][arch]), + build[section][module]["HEX_FLAGS"], + app_file, hex_file, + group=module+'_hex', + after=(module+'_link')) + +def gen_dump(build,section,module,debug): + base_dir = get_base_dir(build,section,module) + arch = build[section][module]['ARCH'] + core = build[section][module]['CORE'] + + print "Making .dump of Application %s/%s" % ( build[section][module]['APP'], + build[section][module]['VERSION'] ) + + app_file = get_destination_file( + get_src_tuple(build,section,module, + build[section][module]['APP'], + debug)) + + dump_file = get_destination_file( + get_src_tuple(build,section,module, + build[section][module]['DUMP'], + debug)) + + # run obj-copy - to generate .hex file + run(join_path(build['TOOLS']['PATH']['SCRIPT'],'capture_stdout'), + join_path(build['TOOLS']['PATH'][arch],build['TOOLS']['OBJ-DUMP'][arch]), + dump_file, + build[section][module]["DUMP_FLAGS"], + app_file, + group=module+'_dump', + after=(module+'_link')) + +def gen_hex2c(build,section,module,debug): + base_dir = get_base_dir(build,section,module) + arch = build[section][module]['ARCH'] + core = build[section][module]['CORE'] + + print "Making .c of Application %s/%s" % ( build[section][module]['APP'], + build[section][module]['VERSION'] ) + + hex_file = get_destination_file( + get_src_tuple(build,section,module, + build[section][module]['HEX'], + debug)) + + c_file = get_destination_file( + get_src_tuple(build,section,module, + build[section][module]['HEX2C'], + debug)) + + # run hex2c - to generate .x file + run(join_path(build['TOOLS']['PATH']['SCRIPT'],'hex2c'), + hex_file, + c_file, + build[section][module]["HEX2C_FLAGS"], + group=module+'_hex2c', + after=(module+'_hex')) + + + +def module_maker(build,section='SRC',debug=False): + + # Dictionaries are inherently unsorted. + # To ensure sources are built in a particular order put an + # "ORDER" key in the source module and number it in ascending + # priority. by default everything is Order 1. they are built + # first in arbitrary order. Then Order 2 and so on. + def module_sorter(module): + order = 1 + if "ORDER" in build[section][module]: + order = build[section][module]["ORDER"] + return order + + for module in sorted(build[section],key=module_sorter): + cmd_cnt = 0; + + if (module not in build['SKIP']): + # Then run any external make commands + if ('MAKE' in build[section][module]): + if ('VCS' in build[section][module]): + after('VCS'+module) + cmd_cnt = execute_make_script(build,section,module,debug) + + + # Then run make type directly from source. + if ('SRC' in build[section][module]) : + if ('VCS' in build[section][module]): + after('VCS'+module) + if ('MAKE' in build[section][module]): + after('MAKE'+module+str(cmd_cnt)) + + src_build(build,section,module,debug) + + # Source files need to be linked. + # Depending on the kind of source, link as appropriate. + # Each option is Mutually Exclusive + if ('LIBRARY' in build[section][module]) : + ar(build,module,debug,section) + elif ('MODULE' in build[section][module]) : + """ + Modules are linked with their APP! + NOTHING TO DO NOW. + """ + elif ('APP' in build[section][module]) : + ld(build,section,module,debug) + + # generate any .hex files as required. + if ('HEX' in build[section][module]): + gen_hex(build,section,module,debug) + + # generate .c files as required. + if ('HEX2C' in build[section][module]): + gen_hex2c(build,section,module,debug) + + # generate .dump files as required. + if ('DUMP' in build[section][module]): + gen_dump(build,section,module,debug) diff --git a/util.py b/util.py new file mode 100644 index 0000000..7c5d42f --- /dev/null +++ b/util.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python + +"""Utility opertations for the Fabricate build tool. + +fabricate is a build tool that finds dependencies automatically for any +language. It's small and just works. No hidden stuff behind your back. It was +inspired by Bill McCloskey's make replacement, memoize, but fabricate works on +Windows as well as Linux. + +These functions are not part of fabricate core, but ease writing build scripts +with it. + +""" +import os +import sys +import subprocess +import atexit +import re + +from fabricate import * + +# Function to flatten an array of arrays to make it easy to generate lists of options. +FLATTEN = lambda z: [x for y in z for x in y] + +# Interleave an array with a set value. Useful for building INCLUDE Definitions. +# Eg, INCS = INTERLEAVE('-I', ['a/b', 'a/c']) +INTERLEAVE = lambda y,z: FLATTEN([[y, x] for x in z]) + +def FILTER(array, excludes): + """ Take an array of items, and remove all occurences of anything in + the exclude array that is found in it. + Then return the filtered array + """ + for element in array: + try: + if excludes.index(element) >= 0: + array.remove(element) + except: + pass + return array + +def get_flag(FLAGS, PATH): + """ Get a specific set of flags from a known path. """ + # Return an empty list when there are no flags. + if FLAGS is None: + return [] + + # If the path is exhausted, return the result as whats left. + if PATH == []: + return FLAGS + + return get_flag(FLAGS.get(PATH[0]), PATH[1:]) + + + +# Recursively iterate through a dictionary defining compiler flags and +# return the set of flags for a particular TOOL, TYPE and DEVICE +def get_flags(FLAGS, TOOL): + # TOOL is a tupple of: + # (TOOL, TYPE, DEVICE), eg. + # ("GCC", "DEBUG", "SAML21G18B") + + # Return an empty list when there are no flags. + if FLAGS is None: + return [] + + # If the flags are an array, then we return the array as the flags. + if not isinstance(FLAGS,dict): + return FLAGS + + # When the flags are a dictionary, we extract the flags from it. + flags = [] + flags = flags + get_flags(FLAGS.get('COMMON'), TOOL ) # COMMON - At all levels + flags = flags + get_flags(FLAGS.get(TOOL[0]+'_FLAGS'), TOOL) # TOOL_FLAGS - Normally only at root + flags = flags + get_flags(FLAGS.get(TOOL[1]), TOOL) # TYPE - At all levels (DEBUG or RELEASE) + + # Special handling for targets. + if isinstance(FLAGS.get("DEVICE"), dict): + flags = flags + get_flags(FLAGS.get("DEVICE").get(TOOL[2]), TOOL) + # DEVICE - Normally at all levels, except root. (Eg. "SAML21G18B") + return flags + +def _mkdir_recursive(path): + # Make a path, use mkdir if we can, so that clean can remove it. + sub_path = os.path.dirname(path) + + if (sub_path <> "") and (not os.path.exists(sub_path)): + _mkdir_recursive(sub_path) + + if not os.path.exists(path): + if sys.platform.startswith('linux'): + # Linux-specific code here... + run("mkdir", "-p", path) + else: + # If building on anything other than linux, + # this should work, but clean wont remove the + # created directories. + os.mkdir(path) + +def out_name(build_dir, source, out_ext): + # Create a Output file name, based on the input. + file_path = os.path.normpath(os.path.join(build_dir, source[0])) + + # Make the build path, if it does not exist + if not os.path.exists(file_path): + _mkdir_recursive(file_path) + + return os.path.join(file_path, source[1]+out_ext) + +def in_name(source): + # Create a Input file name, based on the input. + return os.path.join(source[0], source[1] + source[2]) + +def split_source(source): + source_path, source_name = os.path.split(source) + source_name, source_ext = os.path.splitext(source_name) + + return (source_path, source_name, source_ext) + +def replace_ext(fname,ext): + return os.path.normpath(os.path.splitext(fname)[0]+ext) + +def join_path(*args): + return os.path.normpath(os.path.join(*args)) + +def get_build_dir(build,debug): + if debug: + return build['DEBUG_DIR'] + else: + return build['BUILD_DIR'] + +def get_destination_file(src,new_ext=None,just_dir=False): + strip = src[2]; + dest = src[0]; + + if just_dir: + dest = os.path.dirname(dest) + elif new_ext != None: + dest = replace_ext(dest,new_ext) + + + if strip > 0: + dest = dest.split('/',strip) + dest = dest[len(dest)-1] + + return join_path(src[1],dest) + +def get_base_dir(build,section,module): + if 'BASEDIR' in build[section][module]: + base_dir = build[section][module]['BASEDIR'] + else: + base_dir = build[section][module]['VERSION'] + + if 'PREFIX' in build[section][module]: + base_dir = join_path(build[section][module]['PREFIX'],base_dir) + + return base_dir + +def get_src_tuple(build,section,module,src,debug=False): + # The src tuple is: + # ( "Path to source file", + # "Path to locate destination built file", + # ) + if not isinstance(src, (tuple)): + src = (src, + join_path(get_build_dir(build,debug), + build[section][module]['ARCH'], + build[section][module]['CORE']), + 0) + elif debug: + src = (src[0],join_path(get_build_dir(build,debug),src[1]),src[2]) + + base_dir = get_base_dir(build,section,module) + + src = (join_path(base_dir,src[0]),src[1],src[2]) + return src + +# Returns the name of the tool to build the source file with +def get_tool(build,path): + ext = os.path.splitext(path[0])[1] + for tool in build['EXT']: + if ext in build['EXT'][tool]: + return tool + return "Unknown" + +# Generic Include Path Collector for GCC and G++ +def get_includes(build,section,module,include_uses=True,system=False,debug=False): + base_dir = get_base_dir(build,section,module) + incs = [] + if system: + if 'SYSINCLUDE' in build['SOURCE'][module]: + incs = build['SOURRCE'][module]['SYSINCLUDE'] + else: + if 'INCLUDE' in build['SOURCE'][module]: + incs = build['SOURCE'][module]['INCLUDE'] + + mod_inc = [join_path(base_dir,inc) for inc in incs] + + if include_uses and ('USES' in build['SOURCE'][module]) : + for used_module in build['SOURCE'][module]['USES'] : + mod_inc = mod_inc + get_includes(build,section,used_module,include_uses=False,system=system,debug=debug) + mod_inc = mod_inc + [join_path(get_src_tuple(build,section,used_module,"",debug)[1], + get_base_dir(build,section,used_module))] + + return mod_inc + +def add_option(option, group, current_option): + if option in group: + current_option += group[option] + return current_option + +# Generic Option Collector for GCC and G++ +def get_gcc_opt(build,section,module,tool,arch,core,src,debug=False): + gcc_opt = [] + + if tool in build['OPTS']: + gcc_opt = add_option('WARN',build['OPTS'][tool],gcc_opt) + + if (arch in build['OPTS'][tool]): + gcc_opt = add_option('CFLAGS',build['OPTS'][tool][arch],gcc_opt) + gcc_opt = add_option(core,build['OPTS'][tool][arch],gcc_opt) + if debug: + gcc_opt = add_option('DEBUG_CFLAGS',build['OPTS'][tool][arch],gcc_opt) + else: + gcc_opt = add_option('NONDEBUG_CFLAGS',build['OPTS'][tool][arch],gcc_opt) + else: + print "WARNING: Unknown GCC %s Compiler: %s, can not properly compile: %s [%s]" % (arch, cpu, src[0], module) + + if debug: + gcc_opt = add_option('DEBUG_CFLAGS',build['OPTS'][tool],gcc_opt) + else: + gcc_opt = add_option('CFLAGS',build['OPTS'][tool],gcc_opt) + + gcc_opt = add_option(tool+'_FLAGS',build[section][module],gcc_opt) + gcc_opt = add_option(tool+'_DEFS',build[section][module],gcc_opt) + + # Also get any DEFINES from Used Libraries and Modules + if ('USES' in build[section][module]) : + for used_module in build[section][module]['USES'] : + gcc_opt = add_option(tool+'_DEFS',build[section][used_module],gcc_opt) + + return gcc_opt From 8383d5d331eefe541a6e50a151bef42f53e3ca77 Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Thu, 7 Jun 2018 10:23:56 +0800 Subject: [PATCH 06/14] Add example build script for embedded systems. --- examples/embedded_build.py | 314 +++++++++++++++++++++++++++++++++++++ 1 file changed, 314 insertions(+) create mode 100644 examples/embedded_build.py diff --git a/examples/embedded_build.py b/examples/embedded_build.py new file mode 100644 index 0000000..5be9d79 --- /dev/null +++ b/examples/embedded_build.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python3 +import os +import sys + +from scripts.fabricate import * +#from scripts.build_util import * +from scripts.generators import * +#from scripts.remote_build import * + +####### BUILD TYPE +DEVELOPMENT_BUILD=False + +TARGET={ + "DEVICE" : "NXP_LPCLPC11U24FHI33_301", # Name of device building for. + "CPU" : "cortex-m0", + "CPU_OPTS" : [ "-mcpu=cortex-m0", + "-mtune=cortex-m0", + "-mthumb", + "-DCORE_M0", + "-DINCLUDE_ROM_DIV" + ], + +} + +####### COMMON PROJECT BUILD DEFINITIONS +# Saves Redundancy in BUILD definition. + + +####### PROJECT BUILD DEFINITIONS +# Convention: +# KEYS are Declared 'KEY' +# PARAMS are Declared "PARAM" +BUILD={ + # Build Type Control + 'DEBUG_BUILD' : [True,False], # [True] to build debug-able version. + # [False] to build production version. + # [True,False] to build both a debug-able and production version. + + # Binary Built Files Destination + 'BUILD_DIR' : "build", # Place where Binaries are placed + 'DEBUG_DIR' : "debug_build", # Place where Binaries are placed (debug build) + + # Define which tools are used to build which extensions + 'EXT' : { + 'GCC' : [".c"], + 'GXX' : [".cpp"], + 'GAS' : [".S"], + }, + + # The Tools we use, by architecture. + # 'AVR' toolchain avr-gcc + # 'TOOL' is a Command, typically a code generator, it doesn't belong to a + # tool chain per-se. + # 'SCRIPT' is a Build Script. + 'TOOLS' : { + 'PATH': { + 'ARM' : "/opt/gcc-arm-none-eabi-4_8-2014q1/bin", + 'TOOL' : "../Tools", + 'SCRIPT': "scripts", + }, + 'GCC' : {'ARM' : "arm-none-eabi-gcc"}, + 'GXX' : {'ARM' : "arm-none-eabi-gcc"}, + 'GAS' : {'ARM' : "arm-none-eabi-gcc"}, + 'OBJ-COPY' : {'ARM' : "arm-none-eabi-objcopy"}, + 'OBJ-DUMP' : {'ARM' : "arm-none-eabi-objdump"}, + }, # END TOOLS + + # The Options we pass to the Tools. + 'OPTS' : { + 'GCC' : { + 'WARN' : [ + "-Wall", + ], + 'ARM' : { + 'CFLAGS' : [ + "-std=gnu99", + "-nostartfiles", + "-ffunction-sections", + "-fdata-sections", +# "-fpack-struct", +# "-fno-inline-small-functions", +# "-fno-move-loop-invariants", +# "-fno-tree-scev-cprop", + ] + TARGET['CPU_OPTS'], + 'DEBUG_CFLAGS' : [ + "-DDEBUG", + ], + 'LDFLAGS' : [ + "--relax", + "--gc-sections", + ], + }, + }, # END GCC + 'GXX' : { + 'WARN' : [ + "-Wall", + "-Wno-reorder", + ], + 'AVR' : { + 'CFLAGS' : [ + "-nostartfiles", + "-ffunction-sections", + "-fdata-sections", +# "-fpack-struct", +# "-fno-inline-small-functions", +# "-fno-move-loop-invariants", +# "-fno-tree-scev-cprop", + "-mcpu="+TARGET['CPU'], + "-mtune="+TARGET['CPU'], + ], + 'DEBUG_CFLAGS' : [ + "-DDEBUG", + ], + 'LDFLAGS' : [ + "--relax", + "--gc-sections", + ], + }, + }, # END GXX + 'GAS' : { + 'WARN' : [ + "-Wall", + ], + 'AVR' : { + 'CFLAGS' : [ + "-x","assembler-with-cpp", + "-mcpu="+TARGET['CPU'], + "-mtune="+TARGET['CPU'], +# "-DF_CPU="+str(AVR_TARGET['F_CPU']), + ], + }, + }, # END GAS + }, # END OPTS + + # Targets to skip Building. USED DURING DEVELOPMENT. + 'SKIP' : [ + ], + + # External Sources to Build/Preliminary Operations + # { : { Options }, ... } + # 'VERSION' - Displayed Version of the Module/Library + # 'BASEDIR' - (Optional) Specifies a Directory which contains the source if it is not "VERSION", + # 'MAKE' - List of commands used to "make" these external sources + # + # EXTERNAL is used for: + # 1: to build "build" tools from source + # 2: to fetch or update externally maintained source repositories. + # 3: To generate code files from data. + # 4: Any other prilimary or utility pre-build function + # Technically autotools type configuration tests could be + # integrated at this point. + # All External builds are completed before the main build starts. + 'EXTERNAL' : { + }, + + # Sources to Build + # FORMAT: + # { : { Options }, ... } + # OPTIONS: + # 'VERSION' - Displayed Version of the Module/Library + # 'ARCH' - The Architecture to build the Module/Library with. (Must Match 'TOOLS') + # 'CORE' - The CORE of the processor to build for. (Must Match OPTS[][]) + # 'PREFIX' - Directory Library/Module is found under. + # 'BASEDIR' - (Optional) Specifies a Directory which contains the source if it is not "VERSION", + # 'LIBRARY' - Optional - Package as a Library, called this name. + # 'MODULE' - Optional - Marks a Package as a module of the main application. + # 'APP' - Optional - Marks a Package as the main application. + # 'LISTING' - Optional - Generates an assembler listing of each file. Option is passed to the assembler to control the listing. + # 'SRC' - List of Source Files to Build. + # Each element may be a single file, or a tupple. + # the single file is equivalent to a tuple (file, BUILD[BUILD_DIR], 0) + # the tuple is: + # (src file, dest directory, the number of path elements to strip from source to destination) + # 'INCLUDE' - List of Include Directories specific to Building the Module/Library + # 'USES' - List of Modules/Libraries used by this module + # : Brings in necessary includes and defines from the module. + # + 'SOURCE' : { + 'cexc' : { + 'VERSION' : 'V1.00', + 'ARCH' : "ARM", + 'CORE' : TARGET['DEVICE'], + 'PREFIX' : "", + 'BASEDIR' : "", + 'APP' : "cexc.elf", + 'MAP' : "cexc.map", + 'DUMP' : "cexc.dump", + 'HEX' : "cexc.hex", +# 'HEX2C' : "cexc.c", + 'HEX_FLAGS' : [ + "-j",".text", + "-j",".data", + "-O","ihex", + ], +# 'HEX2C_FLAGS' : [ +# "bootloader","-AVR","-16", +# ], + 'DUMP_FLAGS' : [ + "-xdSs", + ], + 'LISTING' : "-ahls", +# 'COMMON_DEFS' : [ +# "-DF_CPU="+str(AVR_TARGET['F_CPU_BOOT']), +# "-DBOOTLOADER_ADDRESS="+str(AVR_TARGET['BOOTLOADER_ADDRESS']), +# ], + 'DEBUG_DEFS' : [ + "-DDEBUG", + ], + 'GCC_FLAGS' : [ + "-Os", + "-nostdlib", + ], + 'LDFLAGS' : [ +# "--section-start=.text=%X" % AVR_TARGET['BOOTLOADER_ADDRESS'], + "-T","ld/"+TARGET["DEVICE"]+".ld", + ], + 'SRC' : [ + 'app/cr_startup_lpc11xx.c', + 'app/sysinit.c', + 'app/blinky.c', + 'libs/lpc_chip_11uxx_lib/src/sysinit_11xx.c', + 'libs/lpc_chip_11uxx_lib/src/sysctl_11xx.c', + 'libs/lpc_chip_11uxx_lib/src/chip_11xx.c', + 'libs/lpc_chip_11uxx_lib/src/timer_11xx.c', + 'libs/lpc_chip_11uxx_lib/src/clock_11xx.c', + 'libs/lpc_chip_11uxx_lib/src/romdiv_11xx.c', + ], + 'INCLUDE' : [ + ".", + "libs/lpc_chip_11uxx_lib/inc/", +# "configuration/"+AVR_TARGET["DEVICE"]+"/"+AVR_TARGET["BOARD"], + ], +# 'CHECK_BOOTLOADER_ADDRESS' : True, +# 'ORDER' : 1, + }, + }, + + 'DOCS' : ["docs/Doxyfile_html"], +} + +### Main Build Steps +def external(): + module_maker(BUILD,'EXTERNAL',debug=False) + +def compile(): + for debug in BUILD['DEBUG_BUILD']: + module_maker(BUILD,'SOURCE',debug) + +def chk_bootloader(): + if False in BUILD['DEBUG_BUILD']: + debug = False + else: + debug = True + + for module in BUILD['SOURCE']: + if 'CHECK_BOOTLOADER_ADDRESS' in BUILD['SOURCE'][module]: + app_file = get_destination_file( + get_src_tuple(BUILD,'SOURCE',module, + BUILD['SOURCE'][module]['APP'], + debug)) + run("scripts/chk_bootloader", + app_file, + AVR_TARGET["BOOTLOADER_ADDRESS"], + AVR_DEVICES[AVR_TARGET["DEVICE"]]["MAX_FLASH"], + AVR_DEVICES[AVR_TARGET["DEVICE"]]["FLASH_PAGE"]) + +# hexadecimal address for bootloader section to begin. To calculate the best value: +# - make clean; make main.hex; ### output will list data: 2124 (or something like that) +# - for the size of your device (8kb = 1024 * 8 = 8192) subtract above value 2124... = 6068 +# - How many pages in is that? 6068 / 64 (tiny85 page size in bytes) = 94.8125 +# - round that down to 94 - our new bootloader address is 94 * 64 = 6016, in hex = 1780 + +def package(): + return + +def project(): + compile() + after() + chk_bootloader() + package() + +def docs(): + """ + for doc in BUILD['DOCS']: + run(BUILD['TOOLS']['DOC']['GEN'], + doc, + group=doc+'_docs') + """ + return + +## Custom Operations + +def build(): + external() + after() # Do not proceed to build source until external builds finish. + project() + after() + docs() + +def debug(): + BUILD['DEBUG_BUILD'] = [True] + build() + +def default(): + build() + +def clean(): + autoclean() + +def usage(): + help(scripts.fabricate) + +if __name__ == "__main__": + main(parallel_ok=False,debug=True ) From 43465dfbab781447df4ce7e8e6230012e2fbf1e2 Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Thu, 7 Jun 2018 10:24:20 +0800 Subject: [PATCH 07/14] preserve vscode development state --- .vscode/settings.json | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..615aafb --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.pythonPath": "/usr/bin/python3" +} \ No newline at end of file From c63bb12665e8c746fe6b50db9d24239440ac896d Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Wed, 13 Jun 2018 21:57:37 +0800 Subject: [PATCH 08/14] Add utility scripts --- scripts/capture_stdout | 40 ++++++++++++ scripts/chk_bootloader | 62 +++++++++++++++++++ scripts/hex2c | 135 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 237 insertions(+) create mode 100755 scripts/capture_stdout create mode 100755 scripts/chk_bootloader create mode 100755 scripts/hex2c diff --git a/scripts/capture_stdout b/scripts/capture_stdout new file mode 100755 index 0000000..2eddbe2 --- /dev/null +++ b/scripts/capture_stdout @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +# Capture the output of a command, when the output goes to stdout and direct to a file. + +import os,sys,subprocess + +def capture_stdout(command, outputname, args): + output = subprocess.check_output(command + ' ' + ' '.join(args) + " ; exit 0;", + stderr=subprocess.STDOUT, shell=True, universal_newlines=True) + + outfile = open(outputname, 'w') + outfile.write(output) + +def usage(argv): + need_usage = False + if argv is None: + argv = sys.argv + + if ("-h" in argv[1:]) or ("--help" in argv[1:]) or (len(argv) < 3) : + need_usage = True + else: + try: + command = argv[1] + outputname = argv[2] + args = argv[3:] + except: + need_usage = True + + if need_usage: + print ("USAGE: %s ... + +import os,sys,subprocess + +def chk_bootloader(filename, load_address, max_flash, page_size): + size = subprocess.check_output("avr-size "+filename+" ; exit 0;", + stderr=subprocess.STDOUT, shell=True) + size = int(size.split('\n')[1].split()[0]) + bestaddr = ((max_flash-size)/page_size)*page_size + + print "TARGET %s ::--" % filename + if load_address + size > max_flash: + print "\t\t :: ERROR: FLASH SIZE OF %dK EXCEEDED." % (max_flash/1024) + print "\t\t :: \t Either Reduce size of Bootloader by %d bytes." % ((load_address+size)-max_flash) + print "\t\t :: \t OR change BOOTLOADER_ADDRESS to 0x%04X." % (bestaddr) + sys.exit(3) + + if bestaddr == load_address: + print "\t\t :: BOOTLOADER_ADDRESS of 0x%04X ALREADY OPTIMAL." % (load_address) + else: + print "\t\t :: BOOTLOADER_ADDRESS should be 0x%04X not 0x%04X -- WARNING %0d FLASH BYTES WASTED." % (bestaddr, load_address, bestaddr-load_address) + + print "\t\t :: %d BYTES AVAILABLE FOR APPLICATION" % (bestaddr) + print "\t\t :: BOOTLOADER CONSUMES %d PAGES (%0.2f%%) OF FLASH" % ((max_flash-load_address)/page_size,(100.0/float(max_flash))*float(max_flash-load_address)) + print "\t\t :: %d BYTES LEFT IN BOOTLOADER AREA FOR BOOTLOADER GROWTH" % ((max_flash-load_address) - size) + +def usage(argv): + need_usage = False + if argv is None: + argv = sys.argv + if ("-h" in argv[1:]) or ("--help" in argv[1:]) or (len(argv) < 5) : + need_usage = True + else: + try: + filename = argv[1] + if not os.path.isfile(filename): + print "ERROR: File %s NOT FOUND" % filename + need_usage = True + load_address = int(argv[2]) + max_flash = int(argv[3]) + page_size = int(argv[4]) + except: + need_usage = True + + if need_usage: + print "USAGE: %s " % argv[0] + sys.exit(2) + + return (filename, load_address, max_flash, page_size) + +def main(argv=None): + args = usage(argv) + chk_bootloader(args[0],args[1],args[2],args[3]) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/hex2c b/scripts/hex2c new file mode 100755 index 0000000..44b6582 --- /dev/null +++ b/scripts/hex2c @@ -0,0 +1,135 @@ +#!/usr/bin/env python2 + +# Stand-alone intel hex to C code array generator. +# Parameters: +# hex2c +# -AVR makes the file use PROGMEM to mark the array as stored in code space. +# -16 use word array not byte array. +# -8 use byte array not word array. (Default) + +import os,sys,subprocess +from datetime import date + +isAVR = False +isWORD = False + +def hex2c(filename, cname, arrayname): + global isAVR, isWORD + + cdata={} + min_address = 0xffff + max_address = 0x0000 + for line in tuple(open(filename, 'r')): + line = line.strip() + linebytes = [line[i:i+2] for i in xrange(1, len(line)-1, 2)] + if (len(linebytes) >= 5) and (line[0] == ':') and linebytes[3] == '00': + count = int(linebytes[0],16) + address = int(line[3:7],16) + if address < min_address: + min_address = address + chk = 0 + for byte in linebytes: + chk += int(byte,16) + if chk & 0xFF != 0: + print "WARNING: Line %s - FAILED CHECKSUM" % line + for byte in linebytes[4:count+4]: + cdata[address] = byte + address += 1 + if address > max_address: + max_address = address + + cfile = open(cname, 'w') + + arraymark = "PROGMEM " if isAVR else "" + + cfile.write("/*!\n") + cfile.write(" * \\file %s\n" % os.path.basename(cname)) + cfile.write(" *\n") + cfile.write(" * \\author Auto Generated by %s\n" % os.path.basename(__file__)) + cfile.write(" * \\date %s\n" % date.today().ctime()) + cfile.write(" *\n") + cfile.write(" * Generated from %s\n" % os.path.basename(filename)) + cfile.write(" */\n") + cfile.write("\n") + cfile.write("#include \n") + if isAVR: + cfile.write("#include \n") + cfile.write("\n") + cfile.write("const uint16_t %s_address %s= 0x%04X;\n" % (arrayname, arraymark, min_address)) + cfile.write("\n") + if isWORD: + cfile.write("const uint16_t %s_data[%d] %s= {" % (arrayname, (max_address-min_address)/2, arraymark)) + for address in xrange(min_address, max_address, 2): + if address in cdata: + byte = cdata[address] + else: + byte = 'FF' + if address+1 in cdata: + byte += cdata[address+1] + else: + byte += 'FF' + + # New Line + if ((address - min_address) % 14) == 0: + cfile.write("\n /* 0x%04X */ " % address) + cfile.write("0x%s" % byte) + if (address < max_address-2): + cfile.write(", ") + else: + cfile.write("const uint8_t %s_data[%d] %s= {" % (arrayname, max_address-min_address, arraymark)) + for address in xrange(min_address, max_address): + if address in cdata: + byte = cdata[address] + else: + byte = 'FF' + + # New Line + if ((address - min_address) % 9) == 0: + cfile.write("\n /* 0x%04X */ " % address) + cfile.write("0x%s" % byte) + if (address < max_address-1): + cfile.write(", ") + + cfile.write("\n};\n\n") + +def usage(argv): + global isAVR, isWORD + need_usage = False + if argv is None: + argv = sys.argv + if '-AVR' in argv: + isAVR = True + argv.remove('-AVR') + if '-16' in argv: + isWORD = True + argv.remove('-16') + if '-8' in argv: + isWORD = False + argv.remove('-8') + + if ("-h" in argv[1:]) or ("--help" in argv[1:]) or (len(argv) != 4) : + need_usage = True + else: + try: + filename = argv[1] + cname = argv[2] + array_name = argv[3] + except: + need_usage = True + + if need_usage: + print "USAGE: %s