Administrator/NANOKVM-MaixCDK-MIRROR
1
0
Fork 1
mirror of https://github.com/sipeed/MaixCDK.git synced 2026-11-09 17:04:38 -06:00
Code Issues Packages Projects Releases 1 Wiki Activity

optimzie downloader

Browse Source
This commit is contained in:
Neucrack
2024-05-11 19:53:01 +08:00
parent 5eec37e8ef
commit 969e5730ce
50 changed files with 991 additions and 729 deletions
Download Patch File Download Diff File Expand all files Collapse all files

157
tools/cmake/build.py
View File

@@ -1,18 +1,158 @@
import subprocess
import os
from multiprocessing import cpu_count
import shutil
import sys
import re
import traceback
try:
curr_dir = os.path.abspath(os.path.dirname(__file__))
except Exception:
print("-- [ERROR] Please upgrate maixtool by pip install -U maixtool")
sys.exit(1)
if curr_dir not in sys.path:
sys.path.insert(0, curr_dir)
from file_downloader import download_extract_files
thread_num = cpu_count()
def parse_kconfigs(config_path, toolchain_config_path):
configs = {}
with open(config_path, "r") as f:
lines = f.readlines()
for line in lines:
line = line.strip()
if line.startswith("#") or line == "":
continue
k,v = line.split("=", 1)
if v == "y":
v = True
configs[k] = v
with open(toolchain_config_path, "r") as f:
lines = f.readlines()
for line in lines:
line = line.strip()
if line.startswith("#") or line == "":
continue
if line.startswith("set("):
k, v = line.split("(", 1)[1].rsplit(")", 1)[0].split(" ", 1)
configs[k] = v
return configs
def get_components_dirs(find_dirs):
components = {}
for dir in find_dirs:
if (not dir) or not os.path.exists(dir):
continue
for name in os.listdir(dir):
path = os.path.join(dir, name, "CMakeLists.txt")
if os.path.exists(path):
components[name] = os.path.dirname(path)
return components
def get_components_files(components, valid_components, kconfigs):
files = {}
for name in valid_components:
py_path = os.path.join(components[name], "component.py")
if not os.path.exists(py_path):
continue
g = {}
with open(py_path, "r", encoding="utf-8") as f:
exec(f.read(), g)
if "add_file_downloads" in g:
try:
files[name] = g["add_file_downloads"](kconfigs)
except Exception as e:
print("\n\n---- ERROR -----")
traceback.print_exc()
print("----------------")
print(f"[ERROR] execute compoent [{name}]'s component.py failed")
sys.exit(1)
return files
def find_valid_components(components):
'''
get project depends(not accurately, just exclude some obvious not depend components)
return valid components name
'''
# get components quire
depends = {}
for name, dir in components.items():
cmakelist = os.path.join(dir, "CMakeLists.txt")
with open(cmakelist, "r", encoding="utf-8") as f:
content = f.read()
match = re.findall(r'list\(APPEND ADD_REQUIREMENTS(.*?)\)', content, re.DOTALL|re.MULTILINE)
match = list(set(" ".join(match).split()))
depends[name] = []
for r in match:
if r in components:
depends[name].append(r)
# find main depends
def get_depend_recursive(name):
d = depends[name]
for r in depends[name]:
d.extend(get_depend_recursive(r))
d = list(set(d))
return d
valid = ["main"]
valid.extend(get_depend_recursive("main"))
return valid
def get_components_find_dirs(configs):
final = []
find_dirs = [ # same as compile.cmake find_components in project macro
configs.get("MAIXCDK_EXTRA_COMPONENTS_PATH", ""),
configs.get("PY_PKG_COMPONENTS_PATH", ""),
configs.get("PY_USR_PKG_COMPONENTS_PATH", ""),
os.path.join(configs["SDK_PATH"], "components"),
os.path.join(configs["SDK_PATH"], "components", "3rd_party"),
configs["PROJECT_PATH"],
os.path.join(configs["PROJECT_PATH"], "components"),
os.path.join(configs["PROJECT_PATH"], "..", "components"),
]
for dir in find_dirs:
if (not dir) or not os.path.exists(dir):
continue
final.append(dir)
return final
def get_all_components_dl_info(find_dirs, kconfigs):
components = get_components_dirs(find_dirs)
valid_components = find_valid_components(components)
files_info = get_components_files(components, valid_components, kconfigs)
return files_info
def rebuild(build_path, configs, verbose):
os.makedirs(build_path, exist_ok=True)
os.chdir(build_path)
if not os.path.exists(os.path.join(build_path, "config", "global_config.mk")):
config_path = os.path.join(build_path, "config", "global_config.mk")
toolchain_config_path = os.path.join(build_path, "config", "toolchain_config.cmake")
if not os.path.exists(config_path):
menuconfig(configs["SDK_PATH"], build_path, configs, False)
# find and download all files, save to sdk_path/dl/pkgs_info.json
all_configs = configs.copy()
kconfigs = parse_kconfigs(config_path, toolchain_config_path)
all_configs.update(kconfigs)
find_dirs = get_components_find_dirs(configs)
files_info = get_all_components_dl_info(find_dirs, all_configs)
for name, files in files_info.items():
if len(files) > 0:
print(f"\n-- Download files for component {name}")
try:
download_extract_files(files)
except Exception as e:
print("\n\n---- ERROR -----")
traceback.print_exc()
print("----------------")
print(f"[ERROR] download for compoent [{name}] failed, please check its component.py")
sys.exit(1)
# cmake
cmd = ["cmake", "-G", configs["CMAKE_GENERATOR"]]
for k, v in configs.items():
cmd.append("-D{}={}".format(k, v))
@@ -31,6 +171,8 @@ def rebuild(build_path, configs, verbose):
code += "#define PROJECT_ID \"{}\"\n".format(configs["PROJECT_ID"])
code += "#endif\n"
f.write(code)
# build
build(build_path, configs, verbose)
def build(build_path, configs, verbose):
@@ -119,3 +261,14 @@ def menuconfig(sdk_path, build_path, configs, gui = True):
os.remove(name)
print("-- clean build dir complete")
if __name__ == "__main__":
cmd = sys.argv[1]
if cmd == "get_valid_components":
find_dirs = sys.argv[2:]
for dir in find_dirs:
if " " in dir:
raise Exception("Path can not contain space or special charactors")
components = get_components_dirs(find_dirs)
valid_components = find_valid_components(components)
print(";".join(valid_components), end="")

89
tools/cmake/compile.cmake
View File

@@ -18,7 +18,7 @@ set(g_dist_ignore_libs "" CACHE INTERNAL "g_dist_ignore_libs")
set(g_link_search_path "" CACHE INTERNAL "g_link_search_path")
set(g_includes "" CACHE INTERNAL "g_includes")
set(g_srcs "" CACHE INTERNAL "g_srcs")
set(g_files_download_info "" CACHE INTERNAL "g_files_download_info")
# set(g_files_download_info "" CACHE INTERNAL "g_files_download_info")
set(g_enabled_components "" CACHE INTERNAL "g_enabled_components")
# Set project dir, so just projec can include this cmake file!!!
@@ -147,7 +147,11 @@ function(register_component)
if(ADD_LINK_SEARCH_PATH)
foreach(path ${ADD_LINK_SEARCH_PATH})
if(NOT EXISTS "${path}")
prepend(lib_full "${component_dir}/" ${path})
if(IS_ABSOLUTE ${lib})
set(lib_full ${lib})
else()
prepend(lib_full "${component_dir}/" ${path})
endif()
if(NOT EXISTS "${lib_full}")
message(FATAL_ERROR "Can not find ${path} or ${lib_full}")
endif()
@@ -168,7 +172,11 @@ function(register_component)
if(ADD_STATIC_LIB)
foreach(lib ${ADD_STATIC_LIB})
if(NOT EXISTS "${lib}")
prepend(lib_full "${component_dir}/" ${lib})
if(IS_ABSOLUTE ${lib})
set(lib_full ${lib})
else()
prepend(lib_full "${component_dir}/" ${lib})
endif()
if(NOT EXISTS "${lib_full}")
get_property(is_generated SOURCE ${lib} PROPERTY GENERATED)
if(NOT is_generated)
@@ -186,7 +194,11 @@ function(register_component)
set(dynamic_libs ${g_dynamic_libs})
foreach(lib ${ADD_DYNAMIC_LIB})
if(NOT EXISTS "${lib}")
prepend(lib_full "${component_dir}/" ${lib})
if(IS_ABSOLUTE ${lib})
set(lib_full ${lib})
else()
prepend(lib_full "${component_dir}/" ${lib})
endif()
if(NOT EXISTS "${lib_full}")
get_property(is_generated SOURCE ${lib} PROPERTY GENERATED)
if(NOT is_generated)
@@ -235,17 +247,17 @@ function(register_component)
add_dependencies(${component_name} ${component_name}_file_depends)
endif()
# Add files to download list to g_files_download_info
set(files ${g_files_download_info})
if(ADD_FILE_DOWNLOADS)
foreach(file ${ADD_FILE_DOWNLOADS})
string(REPLACE "\n" " " file ${file})
string(REPLACE "\r" " " file ${file})
list(APPEND files "${component_name}+${file}")
endforeach()
endif()
# append file to global variable g_files_download_info
set(g_files_download_info ${files} CACHE INTERNAL "g_files_download_info")
# # Add files to download list to g_files_download_info
# set(files ${g_files_download_info})
# if(ADD_FILE_DOWNLOADS)
# foreach(file ${ADD_FILE_DOWNLOADS})
# string(REPLACE "\n" " " file ${file})
# string(REPLACE "\r" " " file ${file})
# list(APPEND files "${component_name}+${file}")
# endforeach()
# endif()
# # append file to global variable g_files_download_info
# set(g_files_download_info ${files} CACHE INTERNAL "g_files_download_info")
endfunction()
function(is_path_component ret param_path)
@@ -545,15 +557,28 @@ macro(project name)
message(STATUS "Components priority config fail ${component_dirs_sorted}, check config file:${component_priority_conf_file}")
endif()
# guess components used
set(cmd COMMAND ${python} -u ${SDK_PATH}/tools/cmake/build.py get_valid_components ${SDK_PATH}/components ${SDK_PATH}/components/3rd_party ${MAIXCDK_EXTRA_COMPONENTS_PATH} ${PY_PKG_COMPONENTS_PATH} ${PY_USR_PKG_COMPONENTS_PATH} ${PROJECT_SOURCE_DIR}/../components ${PROJECT_SOURCE_DIR} ${PROJECT_SOURCE_DIR}/components)
execute_process(${cmd} RESULT_VARIABLE cmd_res OUTPUT_VARIABLE component_valid)
if(NOT cmd_res EQUAL 0)
message(FATAL_ERROR "Get valid components failed")
endif()
message("-- Only call enabled components: ${component_valid}")
# Call CMakeLists.txt
foreach(component_dir ${component_dirs_sorted})
get_filename_component(base_dir ${component_dir} NAME)
add_subdirectory(${component_dir} ${base_dir} EXCLUDE_FROM_ALL)
if(TARGET ${base_dir})
add_dependencies(${base_dir} update_build_info) # add build info dependence
else()
message(STATUS "component ${base_dir} not enabled")
endif()
list(FIND component_valid ${base_dir} INDEX)
if(INDEX EQUAL -1)
message(STATUS "Ignore component ${base_dir}")
else()
add_subdirectory(${component_dir} ${base_dir} EXCLUDE_FROM_ALL)
if(TARGET ${base_dir})
add_dependencies(${base_dir} update_build_info) # add build info dependence
else()
message(STATUS "component ${base_dir} not enabled")
endif()
endif()
endforeach()
# update all global vars according to g_enabled_components
@@ -583,7 +608,7 @@ macro(project name)
set(${var_name} ${new_items} CACHE INTERNAL ${var_name})
endfunction()
update_global_var(g_files_download_info)
# update_global_var(g_files_download_info)
update_global_var(g_includes)
update_global_var(g_srcs)
update_global_var(g_dynamic_libs)
@@ -595,15 +620,15 @@ macro(project name)
endforeach()
# Download files according to g_files_download_info
if(g_files_download_info)
message(STATUS "Start download files")
set(download_cmd COMMAND ${python} -u ${SDK_PATH}/tools/file_downloader.py ${g_files_download_info})
execute_process(${download_cmd} RESULT_VARIABLE cmd_res)
if(NOT cmd_res EQUAL 0)
message(FATAL_ERROR "Download files failed")
endif()
message(STATUS "Download files complete")
endif()
# if(g_files_download_info)
# message(STATUS "Start download files")
# set(download_cmd COMMAND ${python} -u ${SDK_PATH}/tools/file_downloader.py ${g_files_download_info})
# execute_process(${download_cmd} RESULT_VARIABLE cmd_res)
# if(NOT cmd_res EQUAL 0)
# message(FATAL_ERROR "Download files failed")
# endif()
# message(STATUS "Download files complete")
# endif()
# write global variables to ${CMAKE_BINARY_DIR}/config/cmake_global_variables.json
set(write_global_vars_cmd COMMAND ${python} -u ${SDK_PATH}/tools/cmake/save_global_vars.py
@@ -612,7 +637,7 @@ macro(project name)
--var srcs ${g_srcs}
--var dynamic_libs ${g_dynamic_libs}
--var link_search_path ${g_link_search_path}
--var files_download_info ${g_files_download_info}
# --var files_download_info ${g_files_download_info}
)
execute_process(${write_global_vars_cmd} RESULT_VARIABLE cmd_res)
if(NOT cmd_res EQUAL 0)

5
tools/file_downloader.py → tools/cmake/file_downloader.py
View File

@@ -19,7 +19,7 @@ import zipfile
import lzma
import json
sdk_path = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
sdk_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
lock = Lock()
@@ -262,7 +262,7 @@ def download_extract_files(items):
extract_dir = os.path.join(sdk_path, "dl", "extracted", item["path"])
if not os.path.exists(pkg_path):
print("\n-------------------------------------------------------------------")
print("-- Downloading {} from:\n {},\n save to: {}\n you can also download it manually and save to this position{}{}".format(
print("-- Downloading {} from:\n {}\n save to: {}\n you can also download it manually and save to this position{}{}".format(
item["filename"], item["url"], pkg_path,
"\n other urls: {}".format(item["urls"]) if item["urls"] else "",
"\n sites: {}".format(item["sites"]) if item["sites"] else ""))
@@ -286,6 +286,7 @@ def download_extract_files(items):
continue
renamed_files = []
if "rename" in item:
print(item)
for _from, _to in item["rename"].items():
from_path = os.path.join(extract_dir, _from)
to_path = os.path.join(extract_dir, _to)