Squashed 'OpenVPN Adapter/Vendors/openvpn/' changes from 098fd412a..e6d68831a

e6d68831a deps: update mbedTLS to 2.7.0
59de63fa6 cli.cpp: added OPENVPN_REMOTE_OVERRIDE
caf9cf6c1 RedirectPipe: added additional flags for flexibility
68595de4d ClientAPI::RemoteOverride: added error status
37d848ca2 Log lines from C++ exceptions should contain the text "exception"
f05802cf9 Increase server validation of password size to 16KB to support bundling SAML messages.
52e4d4a5f Increase client validation of password/response size to 16KB to support bundling SAML messages.
a0416ed92 [OVPN3-209] win: add dependencies checksum verification
f6eadbc4d [OVPN3-206] Refactor Windows build system
7b30c2f12 [OVPN3-220] proto.hpp: send initial options set on rekeying
33dd2f29e mbedtls: backport fixes for CVE-2018-0487
0912a9b62 [OVPN3-213] build system: mbedtls timing tests
98fa55576 deps: update asio to 1.12.0
620531101 [OVPN3-215] asio: apply external patches
f4a73bde5 [OVPN3-215] asio: rebase external patches on top of our current commit ID
a61cac928 mbedtls: Patches from 2.7 to fix timing test failures
c892f41fb win: tune dependencies build
8a394a37d [OVPN3-213] build system: mbedtls timing tests
0a3dd67da [OVPN3-190] tun linux: add to/from_json methods
44c6cdfdc [OVPN3-206] readme: update Windows build instructions
0edec4a09 [OVPN3-206] win: update directories in VS projects
3d6fd62cb mac build: improve unittest stability
758ae98c6 [OVPN3-209] win: add dependencies checksum verification
a7642ee82 [OVPN3-205] win: apply mbedTLS patches
ac94b6eb7 [OVPN3-206] Refactor Windows build system
c5bc3859e mbedTLS: don't set endpoint twice in conf object
3d5dd9ee3 [OVPN3-199] mac build: do not overwrite DEP_DIR
b713762ba mbedtls: Patches from 2.7 to fix timing test failures
37ab79fa6 tun linux: apply changes from 362acf0
6a7aee2c9 [OVPN3-190] tun: implement persistence
1d2ebb07f [OVPN3-190] tun: move tun creation to separate class
53e33d634 [OVPN3-190] tun: move content of tun to tuncli
85d3377c2 [OVPN3-190] tun: move tun setup methods to separate file
735b985eb i/o layer:  wrap raw pointers embedded in closures
322ae24b5 OptionList: support variadic template parameter pack in constructors
8a012b454 lz4: added namespace and improved error handling
34998e94a zlib: removed verbose parameter
846ed217d OpenSSL: set SSL_MODE_RELEASE_BUFFERS to conserve memory by releasing unneeded buffers
32e3ea117 OptionList: added show_unused_options() method
fe38233a8 Buffer: added typedefs for thread-safe refcounts
b34b6271e compression: added compress_lz4() and decompress_lz4()
755e1a181 linux/core.hpp: added exclude_from_core() function
a7f6fe64f ManClientInstance::Send: added userprop_local_update() virtual method
94526ac19 BufferAllocated: fixed regression in buffer copy
33c16812e [OVPN3-144] mbedTLS: fix support for 4096bit encrypted keys
f249ab4bd [OVPN3-144] build-mbedtls: run make check before compiling
5040aef4c [OVPN3-144] build-mbedtls: apply patches using git-apply instead of patch
8a5e838ab [OVPN3-144] mbedTLS: fix incompatibility with PKI created by OpenSSL 1.1
e7badefd7 proto.hpp/tls-crypt: fix access to ACK IDs during packet validation
73fa974db proto.hpp: print buffer exception in case of packet access error
79ad5eded Estblishing a stable branch
1c5f20ab0 Hide the @ sign in logs if username is empty
01ee1f5a4 Added ClientAPI::Config::retryOnAuthFailed parameter
05880b136 Added ProfileParseLimits::MAX_SERVER_LIST_SIZE and raise limit to 4096
eedee4fa6 cli.cpp: allow -s server override to reference a friendly name
6e350e9f9 Linux tun setup: use LinuxGW46 to obtain gateway info
3e044c6c7 top-level .gitignore was missing a trailing newline
a27355ac7 Use C++11 push_back(std::move(...))) for pushing objects onto vectors
8c3af2704 HostPort::split_host_port: support unix socket filename as an alternative kind of host
14b588c86 asio: added asio_resolver_results_to_string()
fd6e8e9bf AsioPolySock: minor changes to remote_endpoint_str()
06f5e4d71 AsioBoundSocket::Socket: added to_string() method
8fd968532 RemoteList: minor cleanup in set_endpoint_range()
f9fc2f54e BufferAllocated: improve movability
8cb8d52cd string: added first_line() method
a26b1646b AsioPolySock: extend AltRouting support
ef3a40c27 Listen::Item: added AltRouting mode
02e786bc9 write_binary_atomic: support ConstBuffer
6745799c9 fileunix: added read_binary_unix_fast()
5689c2d9c write_binary_unix(): added ConstBuffer variant
2b0e76453 enum_dir: refactor to allow enumeration via lambda
116a5bd5e bufstr: added const_buf_from_string() method
f8ec81413 Buffer: added const_buffer_ref() variant accepting a const argument
ae98aa8b6 AsioPolySock: support AltRouting
8f81479f1 AsioBoundSocket::Socket: support inheritance
9598918e9 ServerProto: added schedule_disconnect() method.
4516cf67b ServerProto: reset CoarseTime object when AsioTimer is canceled
0ffc76a0b Route: implement operator< so Route objects can be used as map/set keys.
c4af9f68b event_loop_wait_barrier: raise default timeout to 30 seconds
d7fe87540 appversion.hpp: rename VERSION -> BUILD_VERSION

git-subtree-dir: OpenVPN Adapter/Vendors/openvpn
git-subtree-split: e6d68831a71131b7d92fbea93d3b9cbe10ba2068

win/build.py

@@ -39,7 +39,7 @@ def build(parms, srcfile, unit_test=False):
     # onto VC command line.
     options = {
         "ovpn3"   : parms['OVPN3'],
-        "tap"     : os.path.join(parms['TAP'], 'src'),
+        "tap" : os.path.join(build_dir(parms), "tap-windows", "src"),
         "tap_component_id" : parms['TAP_WIN_COMPONENT_ID'],
         "asio"    : os.path.join(build_dir(parms), "asio"),
         "mbedtls" : os.path.join(build_dir(parms), "mbedtls"),
@@ -62,7 +62,7 @@ def build(parms, srcfile, unit_test=False):
         options['extra_lib'] += " fwpuclnt.lib"
 
     # Add jsoncpp (optional)
-    if 'jsoncpp' in parms['LIB_VERSIONS']:
+    if parms.get('USE_JSONCPP'):
         options["jsoncpp"] = os.path.join(build_dir(parms), "jsoncpp")
         options['extra_inc'] += " /DHAVE_JSONCPP /I %(jsoncpp)s/dist" % options
         options['extra_lib_path'] += " /LIBPATH:%(jsoncpp)s/dist" % options
@@ -82,12 +82,9 @@ def build(parms, srcfile, unit_test=False):
 
 if __name__ == "__main__":
     import sys
-    from parms import PARMS
 
-    # some parameters might be redefined, like in Jenkins multibranch pipeline case
-    PARMS['BUILD'] = os.environ.get('BUILD', PARMS['BUILD'])
-    PARMS['OVPN3'] = os.environ.get('OVPN3', PARMS['OVPN3'])
+    params = read_params()
 
-    src = src_fn_argv(PARMS, sys.argv[1:])
+    src = src_fn_argv(params, sys.argv[1:])
     unit_test = is_unit_test(sys.argv[1:])
-    build(PARMS, src, unit_test)
+    build(params, src, unit_test)

win/buildep.py

@@ -1,4 +1,6 @@
-import os, re
+import glob
+import os
+import re
 
 from utils import *
 
@@ -14,26 +16,33 @@ def compile_one_file(parms, srcfile, incdirs):
 
 def build_asio(parms):
     print "**************** ASIO"
-    with Cd(build_dir(parms)) as cd:
+    with Cd(build_dir(parms)):
+        asio_ver = parms["ASIO_VERSION"]
+        url = "https://github.com/chriskohlhoff/asio/archive/%s.tar.gz" % asio_ver
+        arch_path = os.path.join(build_dir(parms), download(url))
+        checksum = sha256_checksum(arch_path)
+        if checksum != parms["ASIO_CSUM"]:
+            sys.exit("Checksum mismatch, expected %s, actual %s" % (parms["ASIO_CSUM"], checksum))
         with ModEnv('PATH', "%s\\bin;%s" % (parms.get('GIT'), os.environ['PATH'])):
-            dist = os.path.realpath('asio')
-            rmtree(dist)
-            d = expand('asio', parms['DEP'], parms.get('LIB_VERSIONS'))
-            os.rename(d, dist)
+            extract(arch_path, "gz")
+            rmtree("asio")
+            os.rename("asio-%s" % asio_ver, "asio")
+            rm(arch_path)
 
 def build_mbedtls(parms):
     print "**************** MBEDTLS"
-    with Cd(build_dir(parms)) as cd:
+    with Cd(build_dir(parms)):
+        url = "https://tls.mbed.org/download/%s-apache.tgz" % parms["MBEDTLS_VERSION"]
+        arch_path = os.path.join(build_dir(parms), download(url))
+        checksum = sha256_checksum(arch_path)
+        if checksum != parms["MBEDTLS_CSUM"]:
+            sys.exit("Checksum mismatch, expected %s, actual %s" % (parms["MBEDTLS_CSUM"], checksum))
         with ModEnv('PATH', "%s\\bin;%s" % (parms.get('GIT'), os.environ['PATH'])):
+            extract(arch_path, "gz")
             dist = os.path.realpath('mbedtls')
             rmtree(dist)
-            d = expand('mbedtls', parms['DEP'], parms.get('LIB_VERSIONS'))
-            if d.endswith("-apache"):
-                d = d[:-7]
-            elif d.endswith("-gpl"):
-                d = d[:-4]
-
-            os.rename(d, dist)
+            os.rename(parms["MBEDTLS_VERSION"], dist)
+            rm(arch_path)
 
             # edit mbedTLS config.h
             conf_fn = os.path.join(dist, 'include', 'mbedtls', 'config.h')
@@ -43,6 +52,17 @@ def build_mbedtls(parms):
             with open(conf_fn, 'w') as f:
                 f.write(conf)
 
+            # apply patches
+            unapplicable_patches = ["0005-data_files-pkcs8-v2-add-keys-generated-with-PRF-SHA1.patch"]
+
+            for patch_file in glob.glob(os.path.join(parms.get('OVPN3'), "core", "deps", "mbedtls", "patches", "*.patch")):
+                for unapplicable_patch in unapplicable_patches:
+                    if patch_file.endswith(unapplicable_patch):
+                        print "Skipping %s, 'git apply' doesn't apply it on Windows" % patch_file
+                        break
+                else:
+                    call(["git", "apply", "--whitespace=nowarn", "--ignore-space-change", "--verbose", patch_file], cwd=dist)
+
             # compile the source files
             os.chdir(os.path.join(dist, "library"))
             obj = []
@@ -58,30 +78,56 @@ def build_mbedtls(parms):
 
 def build_lz4(parms):
     print "**************** LZ4"
-    with Cd(build_dir(parms)) as cd:
+    with Cd(build_dir(parms)):
+        url = "https://github.com/lz4/lz4/archive/v%s.tar.gz" % parms["LZ4_VERSION"][4:]
+        arch_name = download(url)
+        checksum = sha256_checksum(arch_name)
+        if checksum != parms["LZ4_CSUM"]:
+            sys.exit("Checksum mismatch, expected %s, actual %s" % (parms["LZ4_CSUM"], checksum))
         with ModEnv('PATH', "%s\\bin;%s" % (parms.get('GIT'), os.environ['PATH'])):
+            extract(arch_name, "gz")
             dist = os.path.realpath('lz4')
             rmtree(dist)
-            d = expand('lz4', parms['DEP'], parms.get('LIB_VERSIONS'))
-            os.rename(d, dist)
+            os.rename(parms["LZ4_VERSION"], dist)
+            rm(arch_name)
             os.chdir(os.path.join(dist, "lib"))
             compile_one_file(parms, "lz4.c", ())
             vc_cmd(parms, r"lib /OUT:lz4.lib lz4.obj")
 
+def build_tap(parms):
+    print "**************** Windows-TAP"
+    with Cd(build_dir(parms)):
+        url = "https://github.com/OpenVPN/tap-windows6/archive/%s.zip" % parms["TAP_VERSION"]
+        arch_name = download(url)
+        checksum = sha256_checksum(arch_name)
+        if checksum != parms["TAP_CSUM"]:
+            sys.exit("Checksum mismatch, expected %s, actual %s" % (parms["TAP_CSUM"], checksum))
+        with ModEnv('PATH', "%s\\bin;%s" % (parms.get('GIT'), os.environ['PATH'])):
+            extract(arch_name, "zip")
+            dist = os.path.realpath('tap-windows')
+            rmtree(dist)
+            os.rename("tap-windows6-%s" % parms["TAP_VERSION"], dist)
+            rm(arch_name)
+
 def build_jsoncpp(parms):
-    if 'jsoncpp' in parms['LIB_VERSIONS']:
-        print "**************** JSONCPP"
-        with Cd(build_dir(parms)) as cd:
-            with ModEnv('PATH', "%s\\bin;%s" % (parms.get('GIT'), os.environ['PATH'])):
-                dist = os.path.realpath('jsoncpp')
-                rmtree(dist)
-                d = expand('jsoncpp', parms['DEP'], parms.get('LIB_VERSIONS'))
-                os.rename(d, dist)
-                os.chdir(dist)
-                call(["python", "amalgamate.py"])
-                os.chdir(os.path.join(dist, "dist"))
-                compile_one_file(parms, "jsoncpp.cpp", (".",))
-                vc_cmd(parms, r"lib /OUT:jsoncpp.lib jsoncpp.obj")
+    print "**************** JSONCPP"
+    with Cd(build_dir(parms)):
+        url = "https://github.com/open-source-parsers/jsoncpp/archive/%s.tar.gz" % parms["JSONCPP_VERSION"]
+        arch_name = download(url)
+        checksum = sha256_checksum(arch_name)
+        if checksum != parms["JSONCPP_CSUM"]:
+            sys.exit("Checksum mismatch, expected %s, actual %s" % (parms["JSONCPP_CSUM"], checksum))
+        with ModEnv('PATH', "%s\\bin;%s" % (parms.get('GIT'), os.environ['PATH'])):
+            dist = os.path.realpath('jsoncpp')
+            rmtree(dist)
+            extract(arch_name, "gz")
+            rm(arch_name)
+            os.rename("jsoncpp-%s" % parms["JSONCPP_VERSION"], dist)
+            os.chdir(dist)
+            call(["python", "amalgamate.py"])
+            os.chdir(os.path.join(dist, "dist"))
+            compile_one_file(parms, "jsoncpp.cpp", (".",))
+            vc_cmd(parms, r"lib /OUT:jsoncpp.lib jsoncpp.obj")
 
 def build_all(parms):
     wipetree(build_dir(parms))
@@ -89,13 +135,7 @@ def build_all(parms):
     build_mbedtls(parms)
     build_lz4(parms)
     build_jsoncpp(parms)
+    build_tap(parms)
 
 if __name__ == "__main__":
-    from parms import PARMS
-
-    # some parameters might be redefined, like in Jenkins multibranch pipeline case
-    PARMS['BUILD'] = os.environ.get('BUILD', PARMS['BUILD'])
-    PARMS['OVPN3'] = os.environ.get('OVPN3', PARMS['OVPN3'])
-    PARMS['ARCH'] = os.environ.get('ARCH', PARMS['ARCH'])
-
-    build_all(PARMS)
+    build_all(read_params())

win/ovpn3-core.vcxproj

@@ -460,7 +460,7 @@
       <Optimization>Disabled</Optimization>
       <SDLCheck>false</SDLCheck>
       <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;NOMINMAX;_WIN32_WINNT=0x0600;USE_ASIO;ASIO_STANDALONE;USE_MBEDTLS;HAVE_LZ4;TAP_WIN_COMPONENT_ID=tap0901;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <AdditionalIncludeDirectories>$(OVPN3_BUILD)\amd64\mbedtls\include;$(OVPN3_TAP_WINDOWS)\src;$(OVPN3_BUILD)\amd64\asio\asio\include;$(OVPN3_BUILD)\amd64\lz4\lib;$(OVPN3_CORE);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(OVPN3_ROOT)\deps\amd64\mbedtls\include;$(OVPN3_ROOT)\deps\amd64\tap-windows\src;$(OVPN3_ROOT)\deps\amd64\asio\asio\include;$(OVPN3_ROOT)\deps\amd64\lz4\lib;$(OVPN3_ROOT)\core;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <SuppressStartupBanner>false</SuppressStartupBanner>
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
       <AdditionalOptions>/bigobj %(AdditionalOptions)</AdditionalOptions>
@@ -468,7 +468,7 @@
     </ClCompile>
     <Link>
       <GenerateDebugInformation>true</GenerateDebugInformation>
-      <AdditionalLibraryDirectories>$(OVPN3_BUILD)\amd64\mbedtls\library;$(OVPN3_BUILD)\amd64\lz4\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+      <AdditionalLibraryDirectories>$(OVPN3_ROOT)\deps\amd64\mbedtls\library;$(OVPN3_ROOT)\deps\amd64\lz4\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
       <AdditionalDependencies>lz4.lib;mbedtls.lib;fwpuclnt.lib;ws2_32.lib;crypt32.lib;iphlpapi.lib;winmm.lib;advapi32.lib;wininet.lib;shell32.lib;ole32.lib;rpcrt4.lib</AdditionalDependencies>
       <ShowProgress>NotSet</ShowProgress>
     </Link>
@@ -499,14 +499,14 @@
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>false</SDLCheck>
       <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;NOMINMAX;_WIN32_WINNT=0x0600;USE_ASIO;ASIO_STANDALONE;USE_MBEDTLS;HAVE_LZ4;TAP_WIN_COMPONENT_ID=tap0901;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <AdditionalIncludeDirectories>$(OVPN3_BUILD)\amd64\mbedtls\include;$(OVPN3_TAP_WINDOWS)\src;$(OVPN3_BUILD)\amd64\asio\asio\include;$(OVPN3_BUILD)\amd64\lz4\lib;$(OVPN3_CORE);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(OVPN3_ROOT)\deps\amd64\mbedtls\include;$(OVPN3_ROOT)\deps\amd64\tap-windows\src;$(OVPN3_ROOT)\deps\amd64\asio\asio\include;$(OVPN3_ROOT)\deps\amd64\lz4\lib;$(OVPN3_ROOT)\core;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
     </ClCompile>
     <Link>
       <GenerateDebugInformation>true</GenerateDebugInformation>
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
       <OptimizeReferences>true</OptimizeReferences>
-      <AdditionalLibraryDirectories>$(OVPN3_BUILD)\amd64\mbedtls\library;$(OVPN3_BUILD)\amd64\lz4\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+      <AdditionalLibraryDirectories>$(OVPN3_ROOT)\deps\amd64\mbedtls\library;$(OVPN3_ROOT)\deps\amd64\lz4\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
       <AdditionalDependencies>lz4.lib;mbedtls.lib;fwpuclnt.lib;ws2_32.lib;crypt32.lib;iphlpapi.lib;winmm.lib;advapi32.lib;wininet.lib;shell32.lib;ole32.lib;rpcrt4.lib;%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
   </ItemDefinitionGroup>

win/parms.py

@@ -1,26 +0,0 @@
-PARMS = {
-    "DEBUG" : False,
-    "STATIC" : True,
-    "OVPN3" : "c:\\src\\ovpn3",
-    "TAP" : "c:\\src\\tap-windows6",
-    "TAP_WIN_COMPONENT_ID" : "tap0901",  # Community: tap0901, Access Server: tapoas
-    "DEP" : "z:\\james\\downloads",
-    "BUILD" : "c:\\src\\ovpn3-build",
-    "PATCH" : "c:\\src\\as\\pyovpn\\patch",
-    "GIT" : "c:\\Program Files (x86)\\Git",
-    "CPP_EXTRA" : "",
-    "MSVC_DIR" : "c:\\Program Files (x86)\\Microsoft Visual Studio 14.0",
-    "ARCH" : "amd64", # one of amd64, x86, or x86_xp (note that x86_xp requires vcvarsall.bat patch)
-    "LIB_VERSIONS" : {
-        'asio'     : "asio-20170227",
-        'mbedtls'  : "mbedtls-2.4.0",
-        'lz4'      : "lz4-1.7.5",
-    },
-    "GTEST_ROOT": ""
-}
-
-try:
-    from parms_local import PARMS as parms_local
-    PARMS.update(parms_local)
-except ImportError:
-    pass

win/utils.py

@@ -1,4 +1,7 @@
 import os, sys, re, stat, shutil, tarfile, zipfile, subprocess
+import requests
+import rfc6266
+import hashlib
 
 j = os.path.join
 
@@ -260,3 +263,51 @@ def zipdir(path, ziph):
     for root, dirs, files in os.walk(path):
         for file in files:
             ziph.write(os.path.join(root, file))
+
+def download(url):
+    print "Downloading %s" % url
+    response = requests.get(url)
+    fname = rfc6266.parse_headers(response.headers['content-disposition']).filename_unsafe
+    with open(fname, "wb") as f:
+        f.write(response.content)
+    return fname
+
+def sha256_checksum(filename, block_size=65536):
+    sha256 = hashlib.sha256()
+    with open(filename, 'rb') as f:
+        for block in iter(lambda: f.read(block_size), b''):
+            sha256.update(block)
+    return sha256.hexdigest()
+
+def read_params():
+    if not os.environ.get('O3'):
+        sys.exit("Missing required O3 env variable")
+
+    params={}
+    params['OVPN3'] = os.environ.get('O3').rstrip()
+    if not os.environ.get('DEP_DIR'):
+        params["BUILD"] = os.path.join(params['OVPN3'], "deps")
+    else:
+        params['BUILD'] = os.environ.get('DEP_DIR').rstrip()
+    params['ARCH'] = os.environ.get('ARCH', 'amd64').rstrip()
+    params['DEBUG'] = os.environ.get('DEBUG')
+    params['STATIC'] = os.environ.get('STATIC')
+    params['MSVC_DIR'] = os.environ.get('MSVC_DIR', 'c:\\Program Files (x86)\\Microsoft Visual Studio 14.0').rstrip()
+    # Community: tap0901, Access Server: tapoas
+    params['TAP_WIN_COMPONENT_ID'] = os.environ.get('TAP_WIN_COMPONENT_ID', 'tap0901')
+    params['CPP_EXTRA'] = os.environ.get('CPP_EXTRA', '').rstrip()
+    if os.environ.get('USE_JSONSPP'):
+        params['USE_JSONCPP'] = True
+    if os.environ.get('USE_JSONSPP'):
+        params['CONNECT'] = True
+    params['GTEST_ROOT'] = os.environ.get('GTEST_ROOT')
+
+    # read versions
+    with open(os.path.join(params['OVPN3'], "core", "deps", "lib-versions")) as f:
+        for l in [line.strip() for line in f if line.strip()]:
+            name, val = l.split("=")
+            if name.startswith("export"):
+                name = name[6:].strip()
+            params[name] = val
+
+    return params