From b55a8004ed4c3d11d0d820480eca4769665ef38b Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Thu, 28 Nov 2019 19:54:57 +0100 Subject: [PATCH] boilerplate: add inital Makefile with run & install targets Add *Makefile* boilerplate useful for python projects. All python tasks are using a virtualenv from ./local/py3 $ make help run - run developer instance install - developer install (./local) uninstall - uninstall (./local) Signed-off-by: Markus Heiser --- .gitignore | 3 + Makefile | 42 ++++++ utils/makefile.include | 128 ++++++++++++++++++ utils/makefile.python | 290 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 463 insertions(+) create mode 100644 Makefile create mode 100644 utils/makefile.include create mode 100644 utils/makefile.python diff --git a/.gitignore b/.gitignore index db20da83..828856f4 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,6 @@ setup.cfg node_modules/ .tx/ + +local/ +searx.egg-info/ diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..42e492d0 --- /dev/null +++ b/Makefile @@ -0,0 +1,42 @@ +# -*- coding: utf-8; mode: makefile-gmake -*- + +PYOBJECTS = searx +PY_SETUP_EXTRAS ?= \[test\] + +include utils/makefile.include +include utils/makefile.python + +all: clean install + +PHONY += help +help: + @echo ' run - run developer instance' + @echo ' install - developer install (./local)' + @echo ' uninstall - uninstall (./local)' + @echo '' + @$(MAKE) -s -f utils/makefile.include make-help + @echo '' + @$(MAKE) -s -f utils/makefile.python python-help + +PHONY += install +install: pyenvinstall + +PHONY += uninstall +uninstall: pyenvuninstall + +PHONY += clean +clean: pyclean + $(call cmd,common_clean) + +PHONY += run +run: pyenvinstall + $(Q) ( \ + sed -i -e "s/debug : False/debug : True/g" ./searx/settings.yml ; \ + sleep 2 ; \ + xdg-open http://127.0.0.1:8888/ ; \ + sleep 3 ; \ + sed -i -e "s/debug : True/debug : False/g" ./searx/settings.yml ; \ + ) & + $(PY_ENV)/bin/python ./searx/webapp.py + +.PHONY: $(PHONY) diff --git a/utils/makefile.include b/utils/makefile.include new file mode 100644 index 00000000..716889c0 --- /dev/null +++ b/utils/makefile.include @@ -0,0 +1,128 @@ +# -*- coding: utf-8; mode: makefile-gmake -*- + +make-help: + @echo ' make V=0|1 [targets] 0 => quiet build (default), 1 => verbose build' + @echo ' make V=2 [targets] 2 => give reason for rebuild of target' + +quiet_cmd_common_clean = CLEAN $@ + cmd_common_clean = \ + rm -rf tests/build ;\ + find . -name '*.orig' -exec rm -f {} + ;\ + find . -name '*.rej' -exec rm -f {} + ;\ + find . -name '*~' -exec rm -f {} + ;\ + find . -name '*.bak' -exec rm -f {} + ;\ + +FMT = cat +ifeq ($(shell which fmt >/dev/null 2>&1; echo $$?), 0) +FMT = fmt +endif + +# MS-Windows +# +# For a minimal *make-environment*, I'am using the gnu-tools from: +# +# - GNU MCU Eclipse Windows Build Tools, which brings 'make', 'rm' etc. +# https://github.com/gnu-mcu-eclipse/windows-build-tools/releases +# +# - git for Windows, which brings 'find', 'grep' etc. +# https://git-scm.com/download/win + + +# normpath +# +# System-dependent normalization of the path name +# +# usage: $(call normpath,/path/to/file) + +normpath = $1 +ifeq ($(OS),Windows_NT) + normpath = $(subst /,\,$1) +endif + + +# stolen from linux/Makefile +# + +ifeq ("$(origin V)", "command line") + KBUILD_VERBOSE = $(V) +endif +ifndef KBUILD_VERBOSE + KBUILD_VERBOSE = 0 +endif + +ifeq ($(KBUILD_VERBOSE),1) + quiet = + Q = +else + quiet=quiet_ + Q = @ +endif + +# stolen from linux/scripts/Kbuild.include +# + +# Convenient variables +comma := , +quote := " +#" this comment is only for emacs highlighting +squote := ' +#' this comment is only for emacs highlighting +empty := +space := $(empty) $(empty) +space_escape := _-_SPACE_-_ + +# Find any prerequisites that is newer than target or that does not exist. +# PHONY targets skipped in both cases. +any-prereq = $(filter-out $(PHONY),$?) $(filter-out $(PHONY) $(wildcard $^),$^) +# +### +# why - tell why a a target got build +# enabled by make V=2 +# Output (listed in the order they are checked): +# (1) - due to target is PHONY +# (2) - due to target missing +# (3) - due to: file1.h file2.h +# (4) - due to command line change +# (5) - due to missing .cmd file +# (6) - due to target not in $(targets) +# (1) PHONY targets are always build +# (2) No target, so we better build it +# (3) Prerequisite is newer than target +# (4) The command line stored in the file named dir/.target.cmd +# differed from actual command line. This happens when compiler +# options changes +# (5) No dir/.target.cmd file (used to store command line) +# (6) No dir/.target.cmd file and target not listed in $(targets) +# This is a good hint that there is a bug in the kbuild file +ifeq ($(KBUILD_VERBOSE),2) +why = \ + $(if $(filter $@, $(PHONY)),- due to target is PHONY, \ + $(if $(wildcard $@), \ + $(if $(strip $(any-prereq)),- due to: $(any-prereq), \ + $(if $(arg-check), \ + $(if $(cmd_$@),- due to command line change, \ + $(if $(filter $@, $(targets)), \ + - due to missing .cmd file, \ + - due to $(notdir $@) not in $$(targets) \ + ) \ + ) \ + ) \ + ), \ + - due to target missing \ + ) \ + ) + +echo-why = $(call escsq, $(strip $(why))) +endif +# +### +# Escape single quote for use in echo statements +escsq = $(subst $(squote),'\$(squote)',$1) +# +# echo command. +# Short version is used, if $(quiet) equals `quiet_', otherwise full one. +echo-cmd = $(if $($(quiet)cmd_$(1)),echo '$(call escsq,$($(quiet)cmd_$(1)))$(echo-why)';) +# +# printing commands +cmd = @$(echo-cmd) $(cmd_$(1)) + diff --git a/utils/makefile.python b/utils/makefile.python new file mode 100644 index 00000000..228eb3f8 --- /dev/null +++ b/utils/makefile.python @@ -0,0 +1,290 @@ +# -*- coding: utf-8; mode: makefile-gmake -*- + +# list of python packages (folders) or modules (files) of this build +PYOBJECTS ?= + +SITE_PYTHON ?=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))site-python +export PYTHONPATH := $(SITE_PYTHON):$$PYTHONPATH + +# folder where the python distribution takes place +PYDIST ?= ./py_dist +# folder where the python intermediate build files take place +PYBUILD ?= ./py_build +# python version to use +PY ?=3 +PYTHON ?= python$(PY) +PIP ?= pip$(PY) +PIP_INST ?= --user + +# https://www.python.org/dev/peps/pep-0508/#extras +#PY_SETUP_EXTRAS ?= \[develop,test\] +PY_SETUP_EXTRAS ?= + +PYDEBUG ?= --pdb +PYLINT_RC ?= .pylintrc + +TEST_FOLDER ?= ./tests +TEST ?= . + +VTENV_OPTS = "--no-site-packages" +PY_ENV = ./local/py$(PY) +PY_ENV_BIN = $(PY_ENV)/bin +PY_ENV_ACT = . $(PY_ENV_BIN)/activate + +ifeq ($(OS),Windows_NT) + PYTHON = python + PY_ENV_BIN = $(PY_ENV)/Scripts + PY_ENV_ACT = $(PY_ENV_BIN)/activate +endif + +ifeq ($(PYTHON),python) + VIRTUALENV = virtualenv +else + VIRTUALENV = virtualenv --python=$(PYTHON) +endif + +ifeq ($(KBUILD_VERBOSE),1) + PIP_VERBOSE = + VIRTUALENV_VERBOSE = +else + PIP_VERBOSE = "-q" + VIRTUALENV_VERBOSE = "-q" +endif + +python-help:: + @echo 'makefile.python:' + @echo ' pyenv | pyenv[un]install' + @echo ' build $(PY_ENV) & [un]install python objects' + @echo ' targts using pyenv $(PY_ENV):' + @echo ' pylint - run pylint *linting*' + @echo ' pytest - run *tox* test on python objects' + @echo ' pydebug - run tests within a PDB debug session' + @echo ' pybuild - build python packages' + @echo ' pyclean - clean intermediate python objects' + @echo ' targets using system users environment:' + @echo ' py[un]install - [un]install python objects in editable mode' + @echo ' upload-pypi - upload $(PYDIST)/* files to PyPi' + @echo 'options:' + @echo ' make PY=2 [targets] => to eval targets with python 2 ($(PY))' + @echo ' make PIP_INST= => to set/unset pip install options ($(PIP_INST))' + @echo ' make TEST=. => choose test from $(TEST_FOLDER) (default "." runs all)' + @echo ' make DEBUG= => target "debug": do not invoke PDB on errors' + @echo ' make PY_SETUP_EXTRAS => also install extras_require from setup.py \[develop,test\]' + @echo ' when using target "pydebug", set breakpoints within py-source by adding::' + @echo ' DEBUG()' + +# ------------------------------------------------------------------------------ +# OS requirements +# ------------------------------------------------------------------------------ + +PHONY += msg-python-exe python-exe +msg-python-exe: + @echo "\n $(PYTHON) is required\n\n\ + Make sure you have $(PYTHON) installed, grab it from\n\ + https://www.python.org or install it from your package\n\ + manager. On debian based OS these requirements are\n\ + installed by::\n\n\ + sudo -H apt-get install $(PYTHON)\n" | $(FMT) + +ifeq ($(shell which $(PYTHON) >/dev/null 2>&1; echo $$?), 1) +python-exe: msg-python-exe + $(error The '$(PYTHON)' command was not found) +else +python-exe: + @: +endif + +msg-pip-exe: + @echo "\n $(PIP) is required\n\n\ + Make sure you have updated pip installed, grab it from\n\ + https://pip.pypa.io or install it from your package\n\ + manager. On debian based OS these requirements are\n\ + installed by::\n\n\ + sudo -H apt-get install python$(PY)-pip\n" | $(FMT) + +ifeq ($(shell which $(PIP) >/dev/null 2>&1; echo $$?), 1) +pip-exe: msg-pip-exe + $(error The '$(PIP)' command was not found) +else +pip-exe: + @: +endif + +PHONY += msg-virtualenv-exe virtualenv-exe +msg-virtualenv-exe: + @echo "\n virtualenv is required\n\n\ + Make sure you have an updated virtualenv installed, grab it from\n\ + https://virtualenv.pypa.io/en/stable/installation/ or install it\n\ + via pip by::\n\n\ + pip install --user https://github.com/pypa/virtualenv/tarball/master\n" | $(FMT) + +ifeq ($(shell which virtualenv >/dev/null 2>&1; echo $$?), 1) +virtualenv-exe: msg-virtualenv-exe + $(error The 'virtualenv' command was not found) +else +virtualenv-exe: + @: +endif + +# ------------------------------------------------------------------------------ +# commands +# ------------------------------------------------------------------------------ + +# $2 path to folder with setup.py, this uses pip from the OS +quiet_cmd_pyinstall = INSTALL $2 + cmd_pyinstall = $(PIP) $(PIP_VERBOSE) install $(PIP_INST) -e $2$(PY_SETUP_EXTRAS) + +# $2 path to folder with setup.py, this uses pip from pyenv (not OS!) +quiet_cmd_pyenvinstall = PYENV install $2 + cmd_pyenvinstall = $(PY_ENV_BIN)/pip $(PIP_VERBOSE) install -e $2$(PY_SETUP_EXTRAS) + +# Uninstall the package. Since pip does not uninstall the no longer needed +# depencies (something like autoremove) the depencies remain. + +# $2 package name to uninstall, this uses pip from the OS. +quiet_cmd_pyuninstall = UNINSTALL $2 + cmd_pyuninstall = $(PIP) $(PIP_VERBOSE) uninstall --yes $2 + +# $2 path to folder with setup.py, this uses pip from pyenv (not OS!) +quiet_cmd_pyenvuninstall = PYENV uninstall $2 + cmd_pyenvuninstall = $(PY_ENV_BIN)/pip $(PIP_VERBOSE) uninstall --yes $2 + +# $2 path to folder where virtualenv take place +quiet_cmd_virtualenv = PYENV usage: $ source ./$@/bin/activate + cmd_virtualenv = \ + if [ ! -d "./$(PY_ENV)" ];then \ + $(VIRTUALENV) $(VIRTUALENV_VERBOSE) $(VTENV_OPTS) $2; \ + else \ + echo " PYENV using virtualenv from $2"; \ + fi + +# $2 path to lint +quiet_cmd_pylint = LINT $@ + cmd_pylint = $(PY_ENV_BIN)/pylint --rcfile $(PYLINT_RC) $2 + +quiet_cmd_pytest = TEST $@ + cmd_pytest = $(PY_ENV_BIN)/tox -vv + +# setuptools, pip, easy_install its a mess full of cracks, a documentation hell +# and broken by design ... all sucks, I really, really hate all this ... aaargh! +# +# About python packaging see `Python Packaging Authority`_. Most of the names +# here are mapped to ``setup(=..., =...)`` arguments in +# ``setup.py``. See `Packaging and distributing projects`_ about ``setup(...)`` +# arguments. If this is all new for you, start with `PyPI Quick and Dirty`_. +# +# Further read: +# +# - pythonwheels_ +# - setuptools_ +# - packaging_ +# - sdist_ +# - installing_ +# +# .. _`Python Packaging Authority`: https://www.pypa.io +# .. _`Packaging and distributing projects`: https://packaging.python.org/guides/distributing-packages-using-setuptools/ +# .. _`PyPI Quick and Dirty`: https://hynek.me/articles/sharing-your-labor-of-love-pypi-quick-and-dirty/ +# .. _pythonwheels: https://pythonwheels.com/ +# .. _setuptools: https://setuptools.readthedocs.io/en/latest/setuptools.html +# .. _packaging: https://packaging.python.org/guides/distributing-packages-using-setuptools/#packaging-and-distributing-projects +# .. _sdist: https://packaging.python.org/guides/distributing-packages-using-setuptools/#source-distributions +# .. _bdist_wheel: https://packaging.python.org/guides/distributing-packages-using-setuptools/#pure-python-wheels +# .. _installing: https://packaging.python.org/tutorials/installing-packages/ +# +quiet_cmd_pybuild = BUILD $@ + cmd_pybuild = $(PY_ENV_BIN)/$(PYTHON) setup.py \ + sdist -d $(PYDIST) \ + bdist_wheel --bdist-dir $(PYBUILD) -d $(PYDIST) + +quiet_cmd_pyclean = CLEAN $@ +# remove 'build' folder since bdist_wheel does not care the --bdist-dir + cmd_pyclean = \ + rm -rf $(PYDIST) $(PYBUILD) ./local ./.tox *.egg-info ;\ + find . -name '*.pyc' -exec rm -f {} + ;\ + find . -name '*.pyo' -exec rm -f {} + ;\ + find . -name __pycache__ -exec rm -rf {} + + +# ------------------------------------------------------------------------------ +# targets +# ------------------------------------------------------------------------------ + +# for installation use the pip from the OS! +PHONY += pyinstall +pyinstall: pip-exe + $(call cmd,pyinstall,.) + +PHONY += pyuninstall +pyuninstall: pip-exe + $(call cmd,pyuninstall,$(PYOBJECTS)) + +# for installation use the pip from PY_ENV (not the OS)! +PHONY += pyenvinstall +pyenvinstall: $(PY_ENV) + $(call cmd,pyenvinstall,.) + +PHONY += pyenvuninstall +pyenvuninstall: $(PY_ENV) + $(call cmd,pyenvuninstall,$(PYOBJECTS)) + +PHONY += pyclean +pyclean: + $(call cmd,pyclean) + +# to build *local* environment, python and virtualenv from the OS is needed! +pyenv: $(PY_ENV) +$(PY_ENV): virtualenv-exe python-exe + $(call cmd,virtualenv,$(PY_ENV)) + @$(PY_ENV_BIN)/pip install $(PIP_VERBOSE) -r requirements.txt + +PHONY += pylint-exe +pylint-exe: $(PY_ENV) + @$(PY_ENV_BIN)/pip $(PIP_VERBOSE) install pylint + +PHONY += pylint +pylint: pylint-exe + $(call cmd,pylint,$(PYOBJECTS)) + +PHONY += pybuild +pybuild: $(PY_ENV) + $(call cmd,pybuild) + +PHONY += pytest +pytest: $(PY_ENV) + $(call cmd,pytest) + +PHONY += pydebug +# set breakpoint with: +# DEBUG() +# e.g. to run tests in debug mode in emacs use: +# 'M-x pdb' ... 'make pydebug' +pydebug: $(PY_ENV) + DEBUG=$(DEBUG) $(PY_ENV_BIN)/pytest $(DEBUG) -v $(TEST_FOLDER)/$(TEST) + +# install / uninstall python objects into virtualenv (PYENV) +pyenv-install: $(PY_ENV) + @$(PY_ENV_BIN)/pip $(PIP_VERBOSE) install -e . + @echo " ACTIVATE $(call normpath,$(PY_ENV_ACT)) " + +pyenv-uninstall: $(PY_ENV) + @$(PY_ENV_BIN)/pip $(PIP_VERBOSE) uninstall --yes . + +# runs python interpreter from ./local/py/bin/python +pyenv-python: pyenv-install + cd ./local; ../$(PY_ENV_BIN)/python -i + +# With 'dependency_links=' setuptools supports dependencies on packages hosted +# on other reposetories then PyPi, see "Packages Not On PyPI" [1]. The big +# drawback is, due to security reasons (I don't know where the security gate on +# PyPi is), this feature is not supported by pip [2]. Thats why an upload to +# PyPi is required and since uploads via setuptools is not recommended, we have +# to imstall / use twine ... its really a mess. +# +# [1] http://python-packaging.readthedocs.io/en/latest/dependencies.html#packages-not-on-pypi +# [2] https://github.com/pypa/pip/pull/1519 + +# https://github.com/pypa/twine +PHONY += upload-pypi +upload-pypi: pyclean pybuild + @$(PY_ENV_BIN)/twine upload $(PYDIST)/* + +.PHONY: $(PHONY)