diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..615aafb --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.pythonPath": "/usr/bin/python3" +} \ No newline at end of file diff --git a/README.md b/README.md index c783a1f..6e5078c 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,8 @@ -# fabricate # +# fabricate3 -[![Build -Status](https://travis-ci.org/SimonAlfie/fabricate.svg?branch=master)](https://travis-ci.org/chriscz/fabricate) +**fabricate3** is a build tool that finds dependencies automatically for any language. It's small and just works. No hidden stuff behind your back. It was inspired by Bill McCloskey's make replacement, memoize, and is decended from Brush Technologies fabricate. Fabricate3 may be compatible with fabricate but that is not a design goal. Fabricate3 is designed to work on python 3.8+ and has no support for python 2. Rather than having any "gotchas" and unlike fabricate, windows is unsupported. -**fabricate** is a build tool that finds dependencies automatically for any language. It's small and just works. No hidden stuff behind your back. It was inspired by Bill McCloskey's make replacement, memoize, but fabricate works on [Windows](https://github.com/SimonAlfie/fabricate/wiki/HowItWorks#windows-issues) as well as Linux. +## Following is the original fabricate documentation: [Get fabricate.py now](https://raw.githubusercontent.com/SimonAlfie/fabricate/master/fabricate.py), learn [how it works](https://github.com/SimonAlfie/fabricate/wiki/HowItWorks), see how to get [in-Python help](https://github.com/SimonAlfie/fabricate/wiki/Help), or discuss it on the [mailing list](http://groups.google.com/group/fabricate-users). diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/embedded_build.py b/examples/embedded_build.py new file mode 100644 index 0000000..5be9d79 --- /dev/null +++ b/examples/embedded_build.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python3 +import os +import sys + +from scripts.fabricate import * +#from scripts.build_util import * +from scripts.generators import * +#from scripts.remote_build import * + +####### BUILD TYPE +DEVELOPMENT_BUILD=False + +TARGET={ + "DEVICE" : "NXP_LPCLPC11U24FHI33_301", # Name of device building for. + "CPU" : "cortex-m0", + "CPU_OPTS" : [ "-mcpu=cortex-m0", + "-mtune=cortex-m0", + "-mthumb", + "-DCORE_M0", + "-DINCLUDE_ROM_DIV" + ], + +} + +####### COMMON PROJECT BUILD DEFINITIONS +# Saves Redundancy in BUILD definition. + + +####### PROJECT BUILD DEFINITIONS +# Convention: +# KEYS are Declared 'KEY' +# PARAMS are Declared "PARAM" +BUILD={ + # Build Type Control + 'DEBUG_BUILD' : [True,False], # [True] to build debug-able version. + # [False] to build production version. + # [True,False] to build both a debug-able and production version. + + # Binary Built Files Destination + 'BUILD_DIR' : "build", # Place where Binaries are placed + 'DEBUG_DIR' : "debug_build", # Place where Binaries are placed (debug build) + + # Define which tools are used to build which extensions + 'EXT' : { + 'GCC' : [".c"], + 'GXX' : [".cpp"], + 'GAS' : [".S"], + }, + + # The Tools we use, by architecture. + # 'AVR' toolchain avr-gcc + # 'TOOL' is a Command, typically a code generator, it doesn't belong to a + # tool chain per-se. + # 'SCRIPT' is a Build Script. + 'TOOLS' : { + 'PATH': { + 'ARM' : "/opt/gcc-arm-none-eabi-4_8-2014q1/bin", + 'TOOL' : "../Tools", + 'SCRIPT': "scripts", + }, + 'GCC' : {'ARM' : "arm-none-eabi-gcc"}, + 'GXX' : {'ARM' : "arm-none-eabi-gcc"}, + 'GAS' : {'ARM' : "arm-none-eabi-gcc"}, + 'OBJ-COPY' : {'ARM' : "arm-none-eabi-objcopy"}, + 'OBJ-DUMP' : {'ARM' : "arm-none-eabi-objdump"}, + }, # END TOOLS + + # The Options we pass to the Tools. + 'OPTS' : { + 'GCC' : { + 'WARN' : [ + "-Wall", + ], + 'ARM' : { + 'CFLAGS' : [ + "-std=gnu99", + "-nostartfiles", + "-ffunction-sections", + "-fdata-sections", +# "-fpack-struct", +# "-fno-inline-small-functions", +# "-fno-move-loop-invariants", +# "-fno-tree-scev-cprop", + ] + TARGET['CPU_OPTS'], + 'DEBUG_CFLAGS' : [ + "-DDEBUG", + ], + 'LDFLAGS' : [ + "--relax", + "--gc-sections", + ], + }, + }, # END GCC + 'GXX' : { + 'WARN' : [ + "-Wall", + "-Wno-reorder", + ], + 'AVR' : { + 'CFLAGS' : [ + "-nostartfiles", + "-ffunction-sections", + "-fdata-sections", +# "-fpack-struct", +# "-fno-inline-small-functions", +# "-fno-move-loop-invariants", +# "-fno-tree-scev-cprop", + "-mcpu="+TARGET['CPU'], + "-mtune="+TARGET['CPU'], + ], + 'DEBUG_CFLAGS' : [ + "-DDEBUG", + ], + 'LDFLAGS' : [ + "--relax", + "--gc-sections", + ], + }, + }, # END GXX + 'GAS' : { + 'WARN' : [ + "-Wall", + ], + 'AVR' : { + 'CFLAGS' : [ + "-x","assembler-with-cpp", + "-mcpu="+TARGET['CPU'], + "-mtune="+TARGET['CPU'], +# "-DF_CPU="+str(AVR_TARGET['F_CPU']), + ], + }, + }, # END GAS + }, # END OPTS + + # Targets to skip Building. USED DURING DEVELOPMENT. + 'SKIP' : [ + ], + + # External Sources to Build/Preliminary Operations + # { : { Options }, ... } + # 'VERSION' - Displayed Version of the Module/Library + # 'BASEDIR' - (Optional) Specifies a Directory which contains the source if it is not "VERSION", + # 'MAKE' - List of commands used to "make" these external sources + # + # EXTERNAL is used for: + # 1: to build "build" tools from source + # 2: to fetch or update externally maintained source repositories. + # 3: To generate code files from data. + # 4: Any other prilimary or utility pre-build function + # Technically autotools type configuration tests could be + # integrated at this point. + # All External builds are completed before the main build starts. + 'EXTERNAL' : { + }, + + # Sources to Build + # FORMAT: + # { : { Options }, ... } + # OPTIONS: + # 'VERSION' - Displayed Version of the Module/Library + # 'ARCH' - The Architecture to build the Module/Library with. (Must Match 'TOOLS') + # 'CORE' - The CORE of the processor to build for. (Must Match OPTS[][]) + # 'PREFIX' - Directory Library/Module is found under. + # 'BASEDIR' - (Optional) Specifies a Directory which contains the source if it is not "VERSION", + # 'LIBRARY' - Optional - Package as a Library, called this name. + # 'MODULE' - Optional - Marks a Package as a module of the main application. + # 'APP' - Optional - Marks a Package as the main application. + # 'LISTING' - Optional - Generates an assembler listing of each file. Option is passed to the assembler to control the listing. + # 'SRC' - List of Source Files to Build. + # Each element may be a single file, or a tupple. + # the single file is equivalent to a tuple (file, BUILD[BUILD_DIR], 0) + # the tuple is: + # (src file, dest directory, the number of path elements to strip from source to destination) + # 'INCLUDE' - List of Include Directories specific to Building the Module/Library + # 'USES' - List of Modules/Libraries used by this module + # : Brings in necessary includes and defines from the module. + # + 'SOURCE' : { + 'cexc' : { + 'VERSION' : 'V1.00', + 'ARCH' : "ARM", + 'CORE' : TARGET['DEVICE'], + 'PREFIX' : "", + 'BASEDIR' : "", + 'APP' : "cexc.elf", + 'MAP' : "cexc.map", + 'DUMP' : "cexc.dump", + 'HEX' : "cexc.hex", +# 'HEX2C' : "cexc.c", + 'HEX_FLAGS' : [ + "-j",".text", + "-j",".data", + "-O","ihex", + ], +# 'HEX2C_FLAGS' : [ +# "bootloader","-AVR","-16", +# ], + 'DUMP_FLAGS' : [ + "-xdSs", + ], + 'LISTING' : "-ahls", +# 'COMMON_DEFS' : [ +# "-DF_CPU="+str(AVR_TARGET['F_CPU_BOOT']), +# "-DBOOTLOADER_ADDRESS="+str(AVR_TARGET['BOOTLOADER_ADDRESS']), +# ], + 'DEBUG_DEFS' : [ + "-DDEBUG", + ], + 'GCC_FLAGS' : [ + "-Os", + "-nostdlib", + ], + 'LDFLAGS' : [ +# "--section-start=.text=%X" % AVR_TARGET['BOOTLOADER_ADDRESS'], + "-T","ld/"+TARGET["DEVICE"]+".ld", + ], + 'SRC' : [ + 'app/cr_startup_lpc11xx.c', + 'app/sysinit.c', + 'app/blinky.c', + 'libs/lpc_chip_11uxx_lib/src/sysinit_11xx.c', + 'libs/lpc_chip_11uxx_lib/src/sysctl_11xx.c', + 'libs/lpc_chip_11uxx_lib/src/chip_11xx.c', + 'libs/lpc_chip_11uxx_lib/src/timer_11xx.c', + 'libs/lpc_chip_11uxx_lib/src/clock_11xx.c', + 'libs/lpc_chip_11uxx_lib/src/romdiv_11xx.c', + ], + 'INCLUDE' : [ + ".", + "libs/lpc_chip_11uxx_lib/inc/", +# "configuration/"+AVR_TARGET["DEVICE"]+"/"+AVR_TARGET["BOARD"], + ], +# 'CHECK_BOOTLOADER_ADDRESS' : True, +# 'ORDER' : 1, + }, + }, + + 'DOCS' : ["docs/Doxyfile_html"], +} + +### Main Build Steps +def external(): + module_maker(BUILD,'EXTERNAL',debug=False) + +def compile(): + for debug in BUILD['DEBUG_BUILD']: + module_maker(BUILD,'SOURCE',debug) + +def chk_bootloader(): + if False in BUILD['DEBUG_BUILD']: + debug = False + else: + debug = True + + for module in BUILD['SOURCE']: + if 'CHECK_BOOTLOADER_ADDRESS' in BUILD['SOURCE'][module]: + app_file = get_destination_file( + get_src_tuple(BUILD,'SOURCE',module, + BUILD['SOURCE'][module]['APP'], + debug)) + run("scripts/chk_bootloader", + app_file, + AVR_TARGET["BOOTLOADER_ADDRESS"], + AVR_DEVICES[AVR_TARGET["DEVICE"]]["MAX_FLASH"], + AVR_DEVICES[AVR_TARGET["DEVICE"]]["FLASH_PAGE"]) + +# hexadecimal address for bootloader section to begin. To calculate the best value: +# - make clean; make main.hex; ### output will list data: 2124 (or something like that) +# - for the size of your device (8kb = 1024 * 8 = 8192) subtract above value 2124... = 6068 +# - How many pages in is that? 6068 / 64 (tiny85 page size in bytes) = 94.8125 +# - round that down to 94 - our new bootloader address is 94 * 64 = 6016, in hex = 1780 + +def package(): + return + +def project(): + compile() + after() + chk_bootloader() + package() + +def docs(): + """ + for doc in BUILD['DOCS']: + run(BUILD['TOOLS']['DOC']['GEN'], + doc, + group=doc+'_docs') + """ + return + +## Custom Operations + +def build(): + external() + after() # Do not proceed to build source until external builds finish. + project() + after() + docs() + +def debug(): + BUILD['DEBUG_BUILD'] = [True] + build() + +def default(): + build() + +def clean(): + autoclean() + +def usage(): + help(scripts.fabricate) + +if __name__ == "__main__": + main(parallel_ok=False,debug=True ) diff --git a/fabricate.py b/fabricate.py index 5110b9b..bcff3ff 100755 --- a/fabricate.py +++ b/fabricate.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Build tool that finds dependencies automatically for any language. @@ -18,18 +18,20 @@ from fabricate import * help(function) +This version ONLY runs on Python 3. + """ -from __future__ import with_statement, print_function, unicode_literals +# from __future__ import with_statement, print_function, unicode_literals # fabricate version number -__version__ = '1.29.3' +__version__ = '3.00.1' # if version of .deps file has changed, we know to not use it deps_version = 2 import atexit -import optparse +import argparse import os import platform import re @@ -41,27 +43,16 @@ import time import threading # NB uses old camelCase names for backward compatibility import traceback -# multiprocessing module only exists on Python >= 2.6 -try: - import multiprocessing -except ImportError: - class MultiprocessingModule(object): - def __getattr__(self, name): - raise NotImplementedError("multiprocessing module not available, can't do parallel builds") - multiprocessing = MultiprocessingModule() +import multiprocessing -# compatibility -PY3 = sys.version_info[0] == 3 -if PY3: - string_types = str - threading_condition = threading.Condition -else: - string_types = basestring +PY_MAJOR_REQUIRED = 3 +PY_MINOR_REQUIRED = 5 -try: - threading_condition = threading._Condition -except ImportError: - threading_condition = threading.Condition +# compatibility +if (sys.version_info.major != PY_MAJOR_REQUIRED) or (sys.version_info.minor < PY_MINOR_REQUIRED): + sys.stderr.write(("fabricate REQUIRES Python %d.%d. Your version = %s ") % + (PY_MAJOR_REQUIRED, PY_MINOR_REQUIRED, sys.version)) + sys.exit() # so you can do "from fabricate import *" to simplify your build script __all__ = ['setup', 'run', 'autoclean', 'main', 'shell', 'fabricate_version', @@ -135,7 +126,7 @@ def args_to_list(args): if isinstance(arg, (list, tuple)): arglist.extend(args_to_list(arg)) else: - if not isinstance(arg, string_types): + if not isinstance(arg, str): arg = str(arg) arglist.append(arg) return arglist @@ -248,6 +239,9 @@ class RunnerUnsupportedException(Exception): pass class Runner(object): + def __init__(self, builder): + self._builder = builder + def __call__(self, *args, **kwargs): """ Run command and return (dependencies, outputs), where dependencies is a list of the filenames of files that the @@ -265,7 +259,7 @@ def ignore(self, name): class AtimesRunner(Runner): def __init__(self, builder): - self._builder = builder + super().__init__(builder) self.atimes = AtimesRunner.has_atimes(self._builder.dirs) if self.atimes == 0: raise RunnerUnsupportedException( @@ -504,10 +498,10 @@ class StraceRunner(Runner): keep_temps = False def __init__(self, builder, build_dir=None): + super().__init__(builder) self.strace_system_calls = StraceRunner.get_strace_system_calls() if self.strace_system_calls is None: raise RunnerUnsupportedException('strace is not available') - self._builder = builder self.temp_count = 0 self.build_dir = os.path.abspath(build_dir or os.getcwd()) @@ -518,7 +512,7 @@ def get_strace_system_calls(): if platform.system() == 'Windows': # even if windows has strace, it's probably a dodgy cygwin one return None - possible_system_calls = ['open','stat', 'stat64', 'lstat', 'lstat64', + possible_system_calls = ['open','openat', 'stat', 'stat64', 'lstat', 'lstat64', 'execve','exit_group','chdir','mkdir','rename','clone','vfork', 'fork','symlink','creat'] valid_system_calls = [] @@ -536,6 +530,7 @@ def get_strace_system_calls(): # Regular expressions for parsing of strace log _open_re = re.compile(r'(?P\d+)\s+open\("(?P[^"]*)", (?P[^,)]*)') + _openat_re = re.compile(r'(?P\d+)\s+openat\(.+?, "(?P[^"]*)", (?P[^,)]*).*\)\s*=\s*(?P-*\d*)') _stat_re = re.compile(r'(?P\d+)\s+l?stat(?:64)?\("(?P[^"]*)", .*') # stat,lstat,stat64,lstat64 _execve_re = re.compile(r'(?P\d+)\s+execve\("(?P[^"]*)", .*') _creat_re = re.compile(r'(?P\d+)\s+creat\("(?P[^"]*)", .*') @@ -576,7 +571,7 @@ def _do_strace(self, args, kwargs, outfile, outname): processes = {} # dictionary of processes (key = pid) unfinished = {} # list of interrupted entries in strace log for line in outfile: - self._match_line(line, processes, unfinished) + self._match_line(line, processes, unfinished) # collect outputs and dependencies from all processes deps = set() @@ -609,6 +604,7 @@ def _match_line(self, line, processes, unfinished): is_output = False open_match = self._open_re.match(line) + openat_match = self._openat_re.match(line) stat_match = self._stat_re.match(line) execve_match = self._execve_re.match(line) creat_match = self._creat_re.match(line) @@ -648,6 +644,12 @@ def _match_line(self, line, processes, unfinished): if 'O_WRONLY' in mode or 'O_RDWR' in mode: # it's an output file if opened for writing is_output = True + elif openat_match: + match = openat_match + mode = match.group('mode') + if 'O_WRONLY' in mode or 'O_RDWR' in mode: + # it's an output file if opened for writing + is_output = True elif stat_match: match = stat_match elif creat_match: @@ -752,7 +754,7 @@ def __call__(self, *args, **kwargs): class AlwaysRunner(Runner): def __init__(self, builder): - pass + super().__init__(builder) def __call__(self, *args, **kwargs): """ Runner that always runs given command, used as a backup in case @@ -766,7 +768,7 @@ class SmartRunner(Runner): """ Smart command runner that uses StraceRunner if it can, otherwise AtimesRunner if available, otherwise AlwaysRunner. """ def __init__(self, builder): - self._builder = builder + super().__init__(builder) try: self._runner = StraceRunner(self._builder) except RunnerUnsupportedException: @@ -930,8 +932,7 @@ def _results_handler( builder, delay=0.01): except ExecutionError as e: r.results = e _groups.set_ok(id, False) - message, data, status = e - printerr("fabricate: " + message) + printerr("fabricate ExecutionError: " + e.args[0]) else: builder.done(r.command, d, o) # save deps r.results = (r.command, d, o) @@ -955,8 +956,8 @@ def _results_handler( builder, delay=0.01): r.results = False _groups.set_ok(a.do.group, False) _groups.dec_count(a.do.group) - elif isinstance(a.do, threading_condition): - # is this only for threading_condition in after()? + elif isinstance(a.do, threading.Condition): + # is this only for threading.Condition in after()? a.do.acquire() # only mark as done if there is no error a.done = no_error @@ -999,7 +1000,7 @@ class Builder(object): def __init__(self, runner=None, dirs=None, dirdepth=100, ignoreprefix='.', ignore=None, hasher=md5_hasher, depsname='.deps', quiet=False, debug=False, inputs_only=False, parallel_ok=False): - """ Initialise a Builder with the given options. + r""" Initialise a Builder with the given options. "runner" specifies how programs should be run. It is either a callable compatible with the Runner class, or a string selecting @@ -1196,7 +1197,7 @@ def memoize(self, command, **kwargs): This function is for compatiblity with memoize.py and is deprecated. Use run() instead. """ - if isinstance(command, string_types): + if isinstance(command, str): args = shlex.split(command) else: args = args_to_list(command) @@ -1204,8 +1205,7 @@ def memoize(self, command, **kwargs): self.run(args, **kwargs) return 0 except ExecutionError as exc: - message, data, status = exc - return status + return exc.args[2] def outofdate(self, func): """ Return True if given build function is out of date. """ @@ -1342,7 +1342,7 @@ def set_runner(self, runner): try: self.runner = self._runner_map[runner](self) except KeyError: - if isinstance(runner, string_types): + if isinstance(runner, str): # For backwards compatibility, allow runner to be the # name of a method in a derived class: self.runner = getattr(self, runner) @@ -1394,7 +1394,8 @@ def setup(builder=None, default=None, **kwargs): _setup_builder = builder _setup_default = default _setup_kwargs = kwargs -setup.__doc__ += '\n\n' + Builder.__init__.__doc__ +setup.__doc__ += '\n\n' + Builder.__init__.__doc__ # pylint: disable=no-member + def _set_default_builder(): """ Set default builder to Builder() instance if it's not yet set. """ @@ -1463,33 +1464,38 @@ def outofdate(command): def parse_options(usage=_usage, extra_options=None, command_line=None): """ Parse command line options and return (parser, options, args). """ - parser = optparse.OptionParser(usage='Usage: %prog '+usage, - version='%prog '+__version__) - parser.disable_interspersed_args() - parser.add_option('-t', '--time', action='store_true', + parser = argparse.ArgumentParser() + parser.add_argument('--version', action='version', version='%(prog)s '+__version__) + parser.add_argument('-t', '--time', action='store_true', help='use file modification times instead of MD5 sums') - parser.add_option('-d', '--dir', action='append', + parser.add_argument('-d', '--dir', action='append', help='add DIR to list of relevant directories') - parser.add_option('-c', '--clean', action='store_true', + parser.add_argument('-c', '--clean', action='store_true', help='autoclean build outputs before running') - parser.add_option('-q', '--quiet', action='store_true', + parser.add_argument('-q', '--quiet', action='store_true', help="don't echo commands, only print errors") - parser.add_option('-D', '--debug', action='store_true', + parser.add_argument('-D', '--debug', action='store_true', help="show debug info (why commands are rebuilt)") - parser.add_option('-k', '--keep', action='store_true', + parser.add_argument('-k', '--keep', action='store_true', help='keep temporary strace output files') - parser.add_option('-j', '--jobs', type='int', + parser.add_argument('-j', '--jobs', type=int, help='maximum number of parallel jobs') + parser.add_argument('actions', nargs=argparse.REMAINDER, + help="The build actions to perform, such as 'clean', 'debug', 'release'") + if extra_options: # add any user-specified options passed in via main() for option in extra_options: - parser.add_option(option) + name_or_flags = option['name_or_flags'] + del option['name_or_flags'] + parser.add_argument(*name_or_flags,**option) + if command_line is not None: - options, args = parser.parse_args(command_line) + options = parser.parse_args(command_line) else: - options, args = parser.parse_args() + options = parser.parse_args() global _parsed_options - _parsed_options = (parser, options, args) + _parsed_options = (parser, options, options.actions) return _parsed_options def fabricate_version(min=None, max=None): @@ -1524,7 +1530,7 @@ def main(globals_dict=None, build_dir=None, extra_options=None, builder=None, and modify the command line passed to the build script. "default" is the default user script function to call, None = 'build' "extra_options" is an optional list of options created with - optparse.make_option(). The pseudo-global variable main.options + argparse.add_argument(). The pseudo-global variable main.options is set to the parsed options list. "kwargs" is any other keyword arguments to pass to the builder """ global default_builder, default_command, _pool @@ -1598,8 +1604,7 @@ def main(globals_dict=None, build_dir=None, extra_options=None, builder=None, sys.exit(1) after() # wait till the build commands are finished except ExecutionError as exc: - message, data, status = exc.args - printerr('fabricate: ' + message) + printerr('fabricate: ' + exc.args[0]) finally: _stop_results.set() # stop the results gatherer so I don't hang if not options.quiet and os.path.abspath(build_dir) != original_path: diff --git a/generators.py b/generators.py new file mode 100644 index 0000000..0e74b9e --- /dev/null +++ b/generators.py @@ -0,0 +1,339 @@ +#!/usr/bin/env python3 +from . import fabricate as FAB +from . import util + +import os +import platform + +# Generic Builder for GCC and G++ +def gcc(build, section, module, tool, arch, core, src, buildtype): + gcc_opt = util.get_gcc_opt(build,section,module,tool,arch,core,src,buildtype) + + # Get Includes + includes = util.get_includes(build,section,module,buildtype) + sysincludes = util.get_includes(build,section,module,system=True,buildtype=buildtype) + + # Add -I to each include path. + inc_opt = [['-I',inc] for inc in includes] + + # Add -i to each system include path. + sysinc_opt = [['-isystem',inc] for inc in sysincludes] + + if 'LISTING' in build[section][module]: + gcc_opt.extend(["-Wa,%s=%s" % (build[section][module]['LISTING'],util.get_destination_file(src,new_ext='.lst'))]) + + # GCC wont make output directories, so ensure there is a place the output can live. + output_file = util.get_destination_file(src,new_ext='.o') + if not os.path.isdir(os.path.dirname(output_file)): + os.makedirs(os.path.dirname(output_file)) + + tool = [util.join_path(build['TOOLS']['PATH'][arch],build['TOOLS'][tool][arch])] + if 'PFX' in build['TOOLS']: + if arch in build['TOOLS']['PFX']: + tool = [build['TOOLS']['PFX'][arch]] + tool + + after_modules=() + if ('USES' in build['SOURCE'][module]) : + for used_module in build['SOURCE'][module]['USES'] : + after_modules = after_modules + (used_module,) + + # run GCC + FAB.run(tool, + gcc_opt, + inc_opt,sysinc_opt, + '-c', src[0], + '-o', output_file, + group=module, + after=after_modules) + +def ar(build,section,module, buildtype): + #base_dir = util.get_base_dir(build,section,module) + arch = build[section][module]['ARCH'] + #core = build[section][module]['CORE'] + + print ("Archiving Library %s of %s" % ( build[section][module]['LIBRARY'], + build[section][module]['VERSION'] )) + + objects = [util.get_destination_file( + util.get_src_tuple(build,section,module,s,buildtype), + new_ext='.o') for s in build[section][module]['SRC']] + + library = util.get_destination_file( + util.get_src_tuple(build,section,module, + build[section][module]['LIBRARY'], + buildtype)) + + output_dir = os.path.dirname(library) + + if not os.path.isdir(output_dir): + os.makedirs(output_dir) + + # Only safe to run when all objects are build AND the directory it belongs in is made. + FAB.run(util.join_path(build['TOOLS']['PATH'][arch], + build['TOOLS']['AR'][arch]), + 'rcs', library, objects, + group=build[section][module]['LIBRARY'], after=(module)) + +def ld(build, section, module, buildtype): + def add_ldflags(ldflags): + ext_ldflags = [] + for ldflag in ldflags: + ext_ldflags.extend(["-Wl,"+ldflag]) + return ext_ldflags + + #base_dir = util.get_base_dir(build,section,module) + arch = build[section][module]['ARCH'] + core = build[section][module]['CORE'] + + print ("Linking Application %s of %s" % ( build[section][module]['APP'], + build[section][module]['VERSION'] )) + + gcc_opt = util.get_gcc_opt(build,section,module,'LD',arch,core,(""),buildtype) + gcc_opt += add_ldflags(util.get_ld_opt(build,section,module,'LD',arch,core,(""),buildtype)) + + searchpaths = [] + if 'LINK' in build[section][module]: + # Make sure we can find link include files. + searchpaths = searchpaths + [os.path.dirname(build[section][module]['LINK'])] + gcc_opt.extend(["-T,",build[section][module]['LINK']]) + + searchpaths = [["-Wl,-L,"+path] for path in searchpaths] + + # Get Objects to link + objects = [util.get_destination_file( + util.get_src_tuple(build,section,module,s,buildtype), + new_ext='.o') for s in build[section][module]['SRC']] + + output_file = util.get_destination_file( + util.get_src_tuple(build,section,module, + build[section][module]['APP'], + buildtype)) + + mapfile = None + if 'MAP' in build[section][module]: + mapfile = "-Wl,-Map="+util.get_destination_file( + util.get_src_tuple(build,section,module, + build[section][module]['MAP'], + buildtype)) + + + # run GCC - to link + FAB.run(util.join_path(build['TOOLS']['PATH'][arch],build['TOOLS']['GCC'][arch]), + gcc_opt, + objects, + mapfile, + searchpaths, + '-o', output_file, + group=module+'_link', + after=(module)) + + +def src_build(build,section,module,buildtype): + # Build a bunch of source files, based on their extensions. + for src in build[section][module]['SRC']: + clean_src = src + src = util.get_src_tuple(build,section,module,clean_src,buildtype) + tool = util.get_tool(build, src) + + if ((tool == 'GCC') or (tool == 'GXX') or (tool == 'GAS')): + gcc(build, + section, + module, + tool, + build[section][module]['ARCH'], + build[section][module]['CORE'], + src, + buildtype) + else: + print ("%s ERROR: Don't know how to compile : %s from %s using %s" % (section, src, module, tool)) + exit(1) + +def gen_hex(build,section,module,buildtype): + #base_dir = util.get_base_dir(build,section,module) + arch = build[section][module]['ARCH'] + #core = build[section][module]['CORE'] + + print ("Making .hex of Application %s/%s" % ( build[section][module]['APP'], + build[section][module]['VERSION'] )) + + app_file = util.get_destination_file( + util.get_src_tuple(build,section,module, + build[section][module]['APP'], + buildtype)) + + hex_file = util.get_destination_file( + util.get_src_tuple(build,section,module, + build[section][module]['HEX'], + buildtype)) + + # run obj-copy - to generate .hex file + FAB.run(util.join_path(build['TOOLS']['PATH'][arch],build['TOOLS']['OBJ-COPY'][arch]), + build[section][module]["HEX_FLAGS"], + app_file, hex_file, + group=module+'_hex', + after=(module+'_link')) + +def gen_bin(build,section,module,buildtype): + #base_dir = util.get_base_dir(build,section,module) + arch = build[section][module]['ARCH'] + #core = build[section][module]['CORE'] + + print ("Making .bin of Application %s/%s" % ( build[section][module]['APP'], + build[section][module]['VERSION'] )) + + app_file = util.get_destination_file( + util.get_src_tuple(build,section,module, + build[section][module]['APP'], + buildtype)) + + bin_file = util.get_destination_file( + util.get_src_tuple(build,section,module, + build[section][module]['BIN'], + buildtype)) + + # run obj-copy - to generate .hex file + FAB.run(util.join_path(build['TOOLS']['PATH'][arch],build['TOOLS']['OBJ-COPY'][arch]), + build[section][module]["BIN_FLAGS"], + app_file, bin_file, + group=module+'_bin', + after=(module+'_link')) + +def gen_uf2(build,section,module,buildtype): + if ('BIN' not in build[section][module]): + print("ERROR: Can not build UF2 file without a BIN File.") + exit(1) + + binfilename = build[section][module]['BIN'] + arch = build[section][module]['ARCH'] + + print ("Making .uf2 of Application %s/%s" % ( binfilename, + build[section][module]['VERSION'] )) + + bin_file = util.get_destination_file( + util.get_src_tuple(build,section,module, + binfilename, + buildtype)) + + uf2_file = util.get_destination_file( + util.get_src_tuple(build,section,module, + build[section][module]['UF2'], + buildtype)) + + # run uf2conv.py - to generate uf2 file (provided by bootloader) + FAB.run(util.join_path(build['TOOLS']['PATH']['UF2CONV'], + build['TOOLS']['UF2CONV'][arch]), + build[section][module]["UF2_FLAGS"], + "-o", uf2_file, bin_file, + group=module+'_uf2', + after=(module+'_bin')) + +def gen_dump(build,section,module,buildtype): + #base_dir = util.get_base_dir(build,section,module) + arch = build[section][module]['ARCH'] + #core = build[section][module]['CORE'] + + print ("Making .dump of Application %s/%s" % ( build[section][module]['APP'], + build[section][module]['VERSION'] )) + + app_file = util.get_destination_file( + util.get_src_tuple(build,section,module, + build[section][module]['APP'], + buildtype)) + + dump_file = util.get_destination_file( + util.get_src_tuple(build,section,module, + build[section][module]['DUMP'], + buildtype)) + + # run obj-dump - to generate dump of the compiled code + FAB.run(util.join_path(build['TOOLS']['PATH']['SCRIPT'],'capture_stdout'), + util.join_path(build['TOOLS']['PATH'][arch],build['TOOLS']['OBJ-DUMP'][arch]), + dump_file, + build[section][module]["DUMP_FLAGS"], + app_file, + group=module+'_dump', + after=(module+'_link')) + +def gen_hex2c(build,section,module,buildtype): + #base_dir = util.get_base_dir(build,section,module) + #arch = build[section][module]['ARCH'] + #core = build[section][module]['CORE'] + + print ("Making .c of Application %s/%s" % ( build[section][module]['APP'], + build[section][module]['VERSION'] )) + + hex_file = util.get_destination_file( + util.get_src_tuple(build,section,module, + build[section][module]['HEX'], + buildtype)) + + c_file = util.get_destination_file( + util.get_src_tuple(build,section,module, + build[section][module]['HEX2C'], + buildtype)) + + # run hex2c - to generate .x file + FAB.run(util.join_path(build['TOOLS']['PATH']['SCRIPT'],'hex2c'), + hex_file, + c_file, + build[section][module]["HEX2C_FLAGS"], + group=module+'_hex2c', + after=(module+'_hex')) + + + +def module_maker(build,section='SRC', buildtype = None): + + # Dictionaries are inherently unsorted. + # To ensure sources are built in a particular order put an + # "ORDER" key in the source module and number it in ascending + # priority. by default everything is Order 1. they are built + # first in arbitrary order. Then Order 2 and so on. + def module_sorter(module): + order = 1 + if "ORDER" in build[section][module]: + order = build[section][module]["ORDER"] + return order + + for module in sorted(build[section],key=module_sorter): + + if (module not in build['SKIP']): + + src_build(build,section,module,buildtype) + + FAB.after() + + # Source files need to be linked. + # Depending on the kind of source, link as appropriate. + # Each option is Mutually Exclusive + if ('LIBRARY' in build[section][module]) : + ar(build,module,buildtype,section) + elif ('MODULE' in build[section][module]) : + """ + Modules are linked with their APP! + NOTHING TO DO NOW. + """ + elif ('APP' in build[section][module]) : + ld(build,section,module,buildtype) + + # generate any .hex files as required. + if ('HEX' in build[section][module]): + gen_hex(build,section,module,buildtype) + + # generate any .bin files as required. + if ('BIN' in build[section][module]): + gen_bin(build,section,module,buildtype) + + # generate .c files as required. + if ('HEX2C' in build[section][module]): + gen_hex2c(build,section,module,buildtype) + + # generate .dump files as required. + if ('DUMP' in build[section][module]): + gen_dump(build,section,module,buildtype) + + FAB.after() + + # generate any .bin files as required. + if ('UF2' in build[section][module]): + gen_uf2(build,section,module,buildtype) diff --git a/scripts/capture_stdout b/scripts/capture_stdout new file mode 100755 index 0000000..2eddbe2 --- /dev/null +++ b/scripts/capture_stdout @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +# Capture the output of a command, when the output goes to stdout and direct to a file. + +import os,sys,subprocess + +def capture_stdout(command, outputname, args): + output = subprocess.check_output(command + ' ' + ' '.join(args) + " ; exit 0;", + stderr=subprocess.STDOUT, shell=True, universal_newlines=True) + + outfile = open(outputname, 'w') + outfile.write(output) + +def usage(argv): + need_usage = False + if argv is None: + argv = sys.argv + + if ("-h" in argv[1:]) or ("--help" in argv[1:]) or (len(argv) < 3) : + need_usage = True + else: + try: + command = argv[1] + outputname = argv[2] + args = argv[3:] + except: + need_usage = True + + if need_usage: + print ("USAGE: %s ... + +import os,sys,subprocess + +def chk_bootloader(filename, load_address, max_flash, page_size): + size = subprocess.check_output("avr-size "+filename+" ; exit 0;", + stderr=subprocess.STDOUT, shell=True) + size = int(size.split('\n')[1].split()[0]) + bestaddr = ((max_flash-size)/page_size)*page_size + + print "TARGET %s ::--" % filename + if load_address + size > max_flash: + print "\t\t :: ERROR: FLASH SIZE OF %dK EXCEEDED." % (max_flash/1024) + print "\t\t :: \t Either Reduce size of Bootloader by %d bytes." % ((load_address+size)-max_flash) + print "\t\t :: \t OR change BOOTLOADER_ADDRESS to 0x%04X." % (bestaddr) + sys.exit(3) + + if bestaddr == load_address: + print "\t\t :: BOOTLOADER_ADDRESS of 0x%04X ALREADY OPTIMAL." % (load_address) + else: + print "\t\t :: BOOTLOADER_ADDRESS should be 0x%04X not 0x%04X -- WARNING %0d FLASH BYTES WASTED." % (bestaddr, load_address, bestaddr-load_address) + + print "\t\t :: %d BYTES AVAILABLE FOR APPLICATION" % (bestaddr) + print "\t\t :: BOOTLOADER CONSUMES %d PAGES (%0.2f%%) OF FLASH" % ((max_flash-load_address)/page_size,(100.0/float(max_flash))*float(max_flash-load_address)) + print "\t\t :: %d BYTES LEFT IN BOOTLOADER AREA FOR BOOTLOADER GROWTH" % ((max_flash-load_address) - size) + +def usage(argv): + need_usage = False + if argv is None: + argv = sys.argv + if ("-h" in argv[1:]) or ("--help" in argv[1:]) or (len(argv) < 5) : + need_usage = True + else: + try: + filename = argv[1] + if not os.path.isfile(filename): + print "ERROR: File %s NOT FOUND" % filename + need_usage = True + load_address = int(argv[2]) + max_flash = int(argv[3]) + page_size = int(argv[4]) + except: + need_usage = True + + if need_usage: + print "USAGE: %s " % argv[0] + sys.exit(2) + + return (filename, load_address, max_flash, page_size) + +def main(argv=None): + args = usage(argv) + chk_bootloader(args[0],args[1],args[2],args[3]) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/hex2c b/scripts/hex2c new file mode 100755 index 0000000..44b6582 --- /dev/null +++ b/scripts/hex2c @@ -0,0 +1,135 @@ +#!/usr/bin/env python2 + +# Stand-alone intel hex to C code array generator. +# Parameters: +# hex2c +# -AVR makes the file use PROGMEM to mark the array as stored in code space. +# -16 use word array not byte array. +# -8 use byte array not word array. (Default) + +import os,sys,subprocess +from datetime import date + +isAVR = False +isWORD = False + +def hex2c(filename, cname, arrayname): + global isAVR, isWORD + + cdata={} + min_address = 0xffff + max_address = 0x0000 + for line in tuple(open(filename, 'r')): + line = line.strip() + linebytes = [line[i:i+2] for i in xrange(1, len(line)-1, 2)] + if (len(linebytes) >= 5) and (line[0] == ':') and linebytes[3] == '00': + count = int(linebytes[0],16) + address = int(line[3:7],16) + if address < min_address: + min_address = address + chk = 0 + for byte in linebytes: + chk += int(byte,16) + if chk & 0xFF != 0: + print "WARNING: Line %s - FAILED CHECKSUM" % line + for byte in linebytes[4:count+4]: + cdata[address] = byte + address += 1 + if address > max_address: + max_address = address + + cfile = open(cname, 'w') + + arraymark = "PROGMEM " if isAVR else "" + + cfile.write("/*!\n") + cfile.write(" * \\file %s\n" % os.path.basename(cname)) + cfile.write(" *\n") + cfile.write(" * \\author Auto Generated by %s\n" % os.path.basename(__file__)) + cfile.write(" * \\date %s\n" % date.today().ctime()) + cfile.write(" *\n") + cfile.write(" * Generated from %s\n" % os.path.basename(filename)) + cfile.write(" */\n") + cfile.write("\n") + cfile.write("#include \n") + if isAVR: + cfile.write("#include \n") + cfile.write("\n") + cfile.write("const uint16_t %s_address %s= 0x%04X;\n" % (arrayname, arraymark, min_address)) + cfile.write("\n") + if isWORD: + cfile.write("const uint16_t %s_data[%d] %s= {" % (arrayname, (max_address-min_address)/2, arraymark)) + for address in xrange(min_address, max_address, 2): + if address in cdata: + byte = cdata[address] + else: + byte = 'FF' + if address+1 in cdata: + byte += cdata[address+1] + else: + byte += 'FF' + + # New Line + if ((address - min_address) % 14) == 0: + cfile.write("\n /* 0x%04X */ " % address) + cfile.write("0x%s" % byte) + if (address < max_address-2): + cfile.write(", ") + else: + cfile.write("const uint8_t %s_data[%d] %s= {" % (arrayname, max_address-min_address, arraymark)) + for address in xrange(min_address, max_address): + if address in cdata: + byte = cdata[address] + else: + byte = 'FF' + + # New Line + if ((address - min_address) % 9) == 0: + cfile.write("\n /* 0x%04X */ " % address) + cfile.write("0x%s" % byte) + if (address < max_address-1): + cfile.write(", ") + + cfile.write("\n};\n\n") + +def usage(argv): + global isAVR, isWORD + need_usage = False + if argv is None: + argv = sys.argv + if '-AVR' in argv: + isAVR = True + argv.remove('-AVR') + if '-16' in argv: + isWORD = True + argv.remove('-16') + if '-8' in argv: + isWORD = False + argv.remove('-8') + + if ("-h" in argv[1:]) or ("--help" in argv[1:]) or (len(argv) != 4) : + need_usage = True + else: + try: + filename = argv[1] + cname = argv[2] + array_name = argv[3] + except: + need_usage = True + + if need_usage: + print "USAGE: %s