Merge commit '84ad2a289f33a43dd71276cc494f337d0fbb3ed6' into feature/update-dependencies

This commit is contained in:
Sergey Abramchuk
2018-04-04 12:34:20 +03:00
81 changed files with 5189 additions and 856 deletions
+5 -8
View File
@@ -39,7 +39,7 @@ def build(parms, srcfile, unit_test=False):
# onto VC command line.
options = {
"ovpn3" : parms['OVPN3'],
"tap" : os.path.join(parms['TAP'], 'src'),
"tap" : os.path.join(build_dir(parms), "tap-windows", "src"),
"tap_component_id" : parms['TAP_WIN_COMPONENT_ID'],
"asio" : os.path.join(build_dir(parms), "asio"),
"mbedtls" : os.path.join(build_dir(parms), "mbedtls"),
@@ -62,7 +62,7 @@ def build(parms, srcfile, unit_test=False):
options['extra_lib'] += " fwpuclnt.lib"
# Add jsoncpp (optional)
if 'jsoncpp' in parms['LIB_VERSIONS']:
if parms.get('USE_JSONCPP'):
options["jsoncpp"] = os.path.join(build_dir(parms), "jsoncpp")
options['extra_inc'] += " /DHAVE_JSONCPP /I %(jsoncpp)s/dist" % options
options['extra_lib_path'] += " /LIBPATH:%(jsoncpp)s/dist" % options
@@ -82,12 +82,9 @@ def build(parms, srcfile, unit_test=False):
if __name__ == "__main__":
import sys
from parms import PARMS
# some parameters might be redefined, like in Jenkins multibranch pipeline case
PARMS['BUILD'] = os.environ.get('BUILD', PARMS['BUILD'])
PARMS['OVPN3'] = os.environ.get('OVPN3', PARMS['OVPN3'])
params = read_params()
src = src_fn_argv(PARMS, sys.argv[1:])
src = src_fn_argv(params, sys.argv[1:])
unit_test = is_unit_test(sys.argv[1:])
build(PARMS, src, unit_test)
build(params, src, unit_test)
+78 -38
View File
@@ -1,4 +1,6 @@
import os, re
import glob
import os
import re
from utils import *
@@ -14,26 +16,33 @@ def compile_one_file(parms, srcfile, incdirs):
def build_asio(parms):
print "**************** ASIO"
with Cd(build_dir(parms)) as cd:
with Cd(build_dir(parms)):
asio_ver = parms["ASIO_VERSION"]
url = "https://github.com/chriskohlhoff/asio/archive/%s.tar.gz" % asio_ver
arch_path = os.path.join(build_dir(parms), download(url))
checksum = sha256_checksum(arch_path)
if checksum != parms["ASIO_CSUM"]:
sys.exit("Checksum mismatch, expected %s, actual %s" % (parms["ASIO_CSUM"], checksum))
with ModEnv('PATH', "%s\\bin;%s" % (parms.get('GIT'), os.environ['PATH'])):
dist = os.path.realpath('asio')
rmtree(dist)
d = expand('asio', parms['DEP'], parms.get('LIB_VERSIONS'))
os.rename(d, dist)
extract(arch_path, "gz")
rmtree("asio")
os.rename("asio-%s" % asio_ver, "asio")
rm(arch_path)
def build_mbedtls(parms):
print "**************** MBEDTLS"
with Cd(build_dir(parms)) as cd:
with Cd(build_dir(parms)):
url = "https://tls.mbed.org/download/%s-apache.tgz" % parms["MBEDTLS_VERSION"]
arch_path = os.path.join(build_dir(parms), download(url))
checksum = sha256_checksum(arch_path)
if checksum != parms["MBEDTLS_CSUM"]:
sys.exit("Checksum mismatch, expected %s, actual %s" % (parms["MBEDTLS_CSUM"], checksum))
with ModEnv('PATH', "%s\\bin;%s" % (parms.get('GIT'), os.environ['PATH'])):
extract(arch_path, "gz")
dist = os.path.realpath('mbedtls')
rmtree(dist)
d = expand('mbedtls', parms['DEP'], parms.get('LIB_VERSIONS'))
if d.endswith("-apache"):
d = d[:-7]
elif d.endswith("-gpl"):
d = d[:-4]
os.rename(d, dist)
os.rename(parms["MBEDTLS_VERSION"], dist)
rm(arch_path)
# edit mbedTLS config.h
conf_fn = os.path.join(dist, 'include', 'mbedtls', 'config.h')
@@ -43,6 +52,17 @@ def build_mbedtls(parms):
with open(conf_fn, 'w') as f:
f.write(conf)
# apply patches
unapplicable_patches = ["0005-data_files-pkcs8-v2-add-keys-generated-with-PRF-SHA1.patch"]
for patch_file in glob.glob(os.path.join(parms.get('OVPN3'), "core", "deps", "mbedtls", "patches", "*.patch")):
for unapplicable_patch in unapplicable_patches:
if patch_file.endswith(unapplicable_patch):
print "Skipping %s, 'git apply' doesn't apply it on Windows" % patch_file
break
else:
call(["git", "apply", "--whitespace=nowarn", "--ignore-space-change", "--verbose", patch_file], cwd=dist)
# compile the source files
os.chdir(os.path.join(dist, "library"))
obj = []
@@ -58,30 +78,56 @@ def build_mbedtls(parms):
def build_lz4(parms):
print "**************** LZ4"
with Cd(build_dir(parms)) as cd:
with Cd(build_dir(parms)):
url = "https://github.com/lz4/lz4/archive/v%s.tar.gz" % parms["LZ4_VERSION"][4:]
arch_name = download(url)
checksum = sha256_checksum(arch_name)
if checksum != parms["LZ4_CSUM"]:
sys.exit("Checksum mismatch, expected %s, actual %s" % (parms["LZ4_CSUM"], checksum))
with ModEnv('PATH', "%s\\bin;%s" % (parms.get('GIT'), os.environ['PATH'])):
extract(arch_name, "gz")
dist = os.path.realpath('lz4')
rmtree(dist)
d = expand('lz4', parms['DEP'], parms.get('LIB_VERSIONS'))
os.rename(d, dist)
os.rename(parms["LZ4_VERSION"], dist)
rm(arch_name)
os.chdir(os.path.join(dist, "lib"))
compile_one_file(parms, "lz4.c", ())
vc_cmd(parms, r"lib /OUT:lz4.lib lz4.obj")
def build_tap(parms):
print "**************** Windows-TAP"
with Cd(build_dir(parms)):
url = "https://github.com/OpenVPN/tap-windows6/archive/%s.zip" % parms["TAP_VERSION"]
arch_name = download(url)
checksum = sha256_checksum(arch_name)
if checksum != parms["TAP_CSUM"]:
sys.exit("Checksum mismatch, expected %s, actual %s" % (parms["TAP_CSUM"], checksum))
with ModEnv('PATH', "%s\\bin;%s" % (parms.get('GIT'), os.environ['PATH'])):
extract(arch_name, "zip")
dist = os.path.realpath('tap-windows')
rmtree(dist)
os.rename("tap-windows6-%s" % parms["TAP_VERSION"], dist)
rm(arch_name)
def build_jsoncpp(parms):
if 'jsoncpp' in parms['LIB_VERSIONS']:
print "**************** JSONCPP"
with Cd(build_dir(parms)) as cd:
with ModEnv('PATH', "%s\\bin;%s" % (parms.get('GIT'), os.environ['PATH'])):
dist = os.path.realpath('jsoncpp')
rmtree(dist)
d = expand('jsoncpp', parms['DEP'], parms.get('LIB_VERSIONS'))
os.rename(d, dist)
os.chdir(dist)
call(["python", "amalgamate.py"])
os.chdir(os.path.join(dist, "dist"))
compile_one_file(parms, "jsoncpp.cpp", (".",))
vc_cmd(parms, r"lib /OUT:jsoncpp.lib jsoncpp.obj")
print "**************** JSONCPP"
with Cd(build_dir(parms)):
url = "https://github.com/open-source-parsers/jsoncpp/archive/%s.tar.gz" % parms["JSONCPP_VERSION"]
arch_name = download(url)
checksum = sha256_checksum(arch_name)
if checksum != parms["JSONCPP_CSUM"]:
sys.exit("Checksum mismatch, expected %s, actual %s" % (parms["JSONCPP_CSUM"], checksum))
with ModEnv('PATH', "%s\\bin;%s" % (parms.get('GIT'), os.environ['PATH'])):
dist = os.path.realpath('jsoncpp')
rmtree(dist)
extract(arch_name, "gz")
rm(arch_name)
os.rename("jsoncpp-%s" % parms["JSONCPP_VERSION"], dist)
os.chdir(dist)
call(["python", "amalgamate.py"])
os.chdir(os.path.join(dist, "dist"))
compile_one_file(parms, "jsoncpp.cpp", (".",))
vc_cmd(parms, r"lib /OUT:jsoncpp.lib jsoncpp.obj")
def build_all(parms):
wipetree(build_dir(parms))
@@ -89,13 +135,7 @@ def build_all(parms):
build_mbedtls(parms)
build_lz4(parms)
build_jsoncpp(parms)
build_tap(parms)
if __name__ == "__main__":
from parms import PARMS
# some parameters might be redefined, like in Jenkins multibranch pipeline case
PARMS['BUILD'] = os.environ.get('BUILD', PARMS['BUILD'])
PARMS['OVPN3'] = os.environ.get('OVPN3', PARMS['OVPN3'])
PARMS['ARCH'] = os.environ.get('ARCH', PARMS['ARCH'])
build_all(PARMS)
build_all(read_params())
@@ -460,7 +460,7 @@
<Optimization>Disabled</Optimization>
<SDLCheck>false</SDLCheck>
<PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;NOMINMAX;_WIN32_WINNT=0x0600;USE_ASIO;ASIO_STANDALONE;USE_MBEDTLS;HAVE_LZ4;TAP_WIN_COMPONENT_ID=tap0901;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(OVPN3_BUILD)\amd64\mbedtls\include;$(OVPN3_TAP_WINDOWS)\src;$(OVPN3_BUILD)\amd64\asio\asio\include;$(OVPN3_BUILD)\amd64\lz4\lib;$(OVPN3_CORE);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>$(OVPN3_ROOT)\deps\amd64\mbedtls\include;$(OVPN3_ROOT)\deps\amd64\tap-windows\src;$(OVPN3_ROOT)\deps\amd64\asio\asio\include;$(OVPN3_ROOT)\deps\amd64\lz4\lib;$(OVPN3_ROOT)\core;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<SuppressStartupBanner>false</SuppressStartupBanner>
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<AdditionalOptions>/bigobj %(AdditionalOptions)</AdditionalOptions>
@@ -468,7 +468,7 @@
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalLibraryDirectories>$(OVPN3_BUILD)\amd64\mbedtls\library;$(OVPN3_BUILD)\amd64\lz4\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
<AdditionalLibraryDirectories>$(OVPN3_ROOT)\deps\amd64\mbedtls\library;$(OVPN3_ROOT)\deps\amd64\lz4\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
<AdditionalDependencies>lz4.lib;mbedtls.lib;fwpuclnt.lib;ws2_32.lib;crypt32.lib;iphlpapi.lib;winmm.lib;advapi32.lib;wininet.lib;shell32.lib;ole32.lib;rpcrt4.lib</AdditionalDependencies>
<ShowProgress>NotSet</ShowProgress>
</Link>
@@ -499,14 +499,14 @@
<IntrinsicFunctions>true</IntrinsicFunctions>
<SDLCheck>false</SDLCheck>
<PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;NOMINMAX;_WIN32_WINNT=0x0600;USE_ASIO;ASIO_STANDALONE;USE_MBEDTLS;HAVE_LZ4;TAP_WIN_COMPONENT_ID=tap0901;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(OVPN3_BUILD)\amd64\mbedtls\include;$(OVPN3_TAP_WINDOWS)\src;$(OVPN3_BUILD)\amd64\asio\asio\include;$(OVPN3_BUILD)\amd64\lz4\lib;$(OVPN3_CORE);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>$(OVPN3_ROOT)\deps\amd64\mbedtls\include;$(OVPN3_ROOT)\deps\amd64\tap-windows\src;$(OVPN3_ROOT)\deps\amd64\asio\asio\include;$(OVPN3_ROOT)\deps\amd64\lz4\lib;$(OVPN3_ROOT)\core;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<AdditionalLibraryDirectories>$(OVPN3_BUILD)\amd64\mbedtls\library;$(OVPN3_BUILD)\amd64\lz4\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
<AdditionalLibraryDirectories>$(OVPN3_ROOT)\deps\amd64\mbedtls\library;$(OVPN3_ROOT)\deps\amd64\lz4\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
<AdditionalDependencies>lz4.lib;mbedtls.lib;fwpuclnt.lib;ws2_32.lib;crypt32.lib;iphlpapi.lib;winmm.lib;advapi32.lib;wininet.lib;shell32.lib;ole32.lib;rpcrt4.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
@@ -1,26 +0,0 @@
PARMS = {
"DEBUG" : False,
"STATIC" : True,
"OVPN3" : "c:\\src\\ovpn3",
"TAP" : "c:\\src\\tap-windows6",
"TAP_WIN_COMPONENT_ID" : "tap0901", # Community: tap0901, Access Server: tapoas
"DEP" : "z:\\james\\downloads",
"BUILD" : "c:\\src\\ovpn3-build",
"PATCH" : "c:\\src\\as\\pyovpn\\patch",
"GIT" : "c:\\Program Files (x86)\\Git",
"CPP_EXTRA" : "",
"MSVC_DIR" : "c:\\Program Files (x86)\\Microsoft Visual Studio 14.0",
"ARCH" : "amd64", # one of amd64, x86, or x86_xp (note that x86_xp requires vcvarsall.bat patch)
"LIB_VERSIONS" : {
'asio' : "asio-20170227",
'mbedtls' : "mbedtls-2.4.0",
'lz4' : "lz4-1.7.5",
},
"GTEST_ROOT": ""
}
try:
from parms_local import PARMS as parms_local
PARMS.update(parms_local)
except ImportError:
pass
@@ -1,4 +1,7 @@
import os, sys, re, stat, shutil, tarfile, zipfile, subprocess
import requests
import rfc6266
import hashlib
j = os.path.join
@@ -260,3 +263,51 @@ def zipdir(path, ziph):
for root, dirs, files in os.walk(path):
for file in files:
ziph.write(os.path.join(root, file))
def download(url):
print "Downloading %s" % url
response = requests.get(url)
fname = rfc6266.parse_headers(response.headers['content-disposition']).filename_unsafe
with open(fname, "wb") as f:
f.write(response.content)
return fname
def sha256_checksum(filename, block_size=65536):
sha256 = hashlib.sha256()
with open(filename, 'rb') as f:
for block in iter(lambda: f.read(block_size), b''):
sha256.update(block)
return sha256.hexdigest()
def read_params():
if not os.environ.get('O3'):
sys.exit("Missing required O3 env variable")
params={}
params['OVPN3'] = os.environ.get('O3').rstrip()
if not os.environ.get('DEP_DIR'):
params["BUILD"] = os.path.join(params['OVPN3'], "deps")
else:
params['BUILD'] = os.environ.get('DEP_DIR').rstrip()
params['ARCH'] = os.environ.get('ARCH', 'amd64').rstrip()
params['DEBUG'] = os.environ.get('DEBUG')
params['STATIC'] = os.environ.get('STATIC')
params['MSVC_DIR'] = os.environ.get('MSVC_DIR', 'c:\\Program Files (x86)\\Microsoft Visual Studio 14.0').rstrip()
# Community: tap0901, Access Server: tapoas
params['TAP_WIN_COMPONENT_ID'] = os.environ.get('TAP_WIN_COMPONENT_ID', 'tap0901')
params['CPP_EXTRA'] = os.environ.get('CPP_EXTRA', '').rstrip()
if os.environ.get('USE_JSONSPP'):
params['USE_JSONCPP'] = True
if os.environ.get('USE_JSONSPP'):
params['CONNECT'] = True
params['GTEST_ROOT'] = os.environ.get('GTEST_ROOT')
# read versions
with open(os.path.join(params['OVPN3'], "core", "deps", "lib-versions")) as f:
for l in [line.strip() for line in f if line.strip()]:
name, val = l.split("=")
if name.startswith("export"):
name = name[6:].strip()
params[name] = val
return params