Compiling Python to C

Return

Cython is a superset of Python that includes a compiler that allows Python code to more easily interface with C code. Cython can provide a significant performance boost to Python code. The compiled code is wrapped in interface code and provides foreign function interfaces for C/C++ libraries. The C files contain:

◦ CPython API calls to create/manipulate Python objects
◦ Type-specific optimizations from Cython
◦ Direct C operations where possible (avoiding Python overhead)
◦ Reference counting management

Creating a program with Cython involves 2 build steps. The first step is to compile the Python code into C code. The second step is to compile the C code into a shared library. Due to this, Cython is also really great for obfuscating Python code. The script I provide here will take an entire Python project, compile it to C, and then package it into an executable using PyInstaller. This was originally designed to compile a Flask project into a single, performant executable.

The script has the following steps:

1. Initial setup and python to C conversion
2. Executable entry point creation
3. PyInstaller spec generation and setup
4. Final executable creation

This script can only output an executable for the platform it is run on. It provides support for including hidden dependencies, building entire folders and skipping certain files.

Python

import os
import glob
import shutil
import logging
from pathlib import Path
from typing import List, Dict
from dataclasses import dataclass

# Required import, extends distutils
import setuptools
from dotenv import load_dotenv
from distutils.core import setup
from Cython.Build import cythonize
from Cython.Compiler import Options
from Cython.Distutils import build_ext
from distutils.extension import Extension


@dataclass
class CompilerConfig:
    threads: int = 8
    version: str = '1.0.0'
    app_name: str = 'Application'
    build_dir: str = 'build'
    compile_dirs: List[str] = []
    exclude_files: List[str] = []
    hidden_imports: List[str] = []
    icon_path: str = 'icon.ico'
    console: bool = True
    additional_binaries: Dict[str, str] = {}
    additional_trees: Dict[str, str] = {}

    def __post_init__(self):
        self.compile_dirs = self.compile_dirs or ['']
        self.exclude_files = self.exclude_files or ['__init__.py']
        self.hidden_imports = self.hidden_imports or []
        self.additional_binaries = self.additional_binaries or {}
        self.additional_trees = self.additional_trees or {}


class PythonCompiler:
    def __init__(self, config: CompilerConfig):
        self.config = config
        self.build_path = Path(config.build_dir)
        self.logger = logging.getLogger(__name__)
        Options.docstrings = False

        # Load environment variables
        load_dotenv()

    def compile(self):
        """Main compilation process"""
        self._setup_build_directory()
        self._create_entry_point()
        self._compile_to_c()
        self._generate_spec_file()
        self._create_executable()

    def _setup_build_directory(self):
        """Prepare build directory"""
        self.build_path.mkdir(exist_ok=True)

    def _create_entry_point(self):
        """Create entry point script"""
        entry_point = self.build_path / 'entry.py'
        entry_point.write_text(
            'from main import main\n' "if __name__ == '__main__':\n" '    main()'
        )

    def _compile_to_c(self):
        """Convert Python files to C using Cython"""
        modules = self._collect_modules()

        setup(
            version=self.config.version,
            name=self.config.app_name,
            cmdclass={'build_ext': build_ext},
            script_args=['build_ext', '--inplace'],
            ext_modules=cythonize(
                list(modules.values()),
                build_dir=str(self.build_path),
                compiler_directives={
                    'always_allow_keywords': True,
                    'language_level': '3',
                    'emit_code_comments': False,
                },
                nthreads=self.config.threads,
            ),
        )

    def _collect_modules(self) -> Dict[str, Extension]:
        """Collect Python modules for compilation"""
        modules = {}
        for pattern in self._build_glob_patterns('py'):
            for py_file in glob.glob(pattern, recursive=True):
                if Path(py_file).name in self.config.exclude_files:
                    continue

                build_dir = str(Path(py_file).with_suffix('')).replace(os.sep, '.')
                if build_dir in modules:
                    modules[build_dir].append(py_file)
                else:
                    modules[build_dir] = [py_file]

        return {
            build_dir: Extension(build_dir, source_files)
            for build_dir, source_files in modules.items()
        }

    def _build_glob_patterns(self, extension: str) -> List[str]:
        """Build glob patterns for file collection"""
        return [
            str(Path(folder) / f'**/*.{extension}' if folder else f'*.{extension}')
            for folder in self.config.compile_dirs
        ]

    def _generate_spec_file(self):
        """Generate PyInstaller spec file"""
        spec_path = self.build_path / f'{self.config.app_name}.spec'

        # Generate initial spec file
        os.system(
            f'cd {self.build_path} && ' f'pyi-makespec entry.py -n {self.config.app_name} --onefile'
        )

        # Modify spec file
        spec_content = spec_path.read_text()
        spec_content = spec_content.replace(
            'hiddenimports=[]', f'hiddenimports={self.config.hidden_imports}'
        ).replace('console=True', f'console={str(self.config.console)}')

        if self.config.icon_path:
            spec_content = spec_content.replace(
                'console=True',
                f'console={str(self.config.console)},\n    icon="{self.config.icon_path}"',
            )

        # Add additional binaries and tree directories
        for binary_src, binary_dst in self.config.additional_binaries.items():
            spec_content = spec_content.replace(
                'a.binaries,',
                f'a.binaries + [(r"{binary_dst}", r"{binary_src}", "BINARY")],',
            )

        for tree_src, tree_prefix in self.config.additional_trees.items():
            spec_content = spec_content.replace(
                'a.binaries,',
                f'a.binaries,\n    Tree(r"{tree_src}", prefix=r"{tree_prefix}"),',
            )

        spec_path.write_text(spec_content)

    def _create_executable(self):
        """Create final executable"""
        os.system(f'cd {self.build_path} && ' f'pyinstaller {self.config.app_name}.spec --clean')

        exe_name = f'{self.config.app_name}.exe' if os.name == 'nt' else self.config.app_name
        exe_path = Path(exe_name)

        if exe_path.exists():
            exe_path.unlink()

        shutil.move(self.build_path / 'dist' / exe_name, exe_name)


if __name__ == '__main__':
    # Example config to compile a Flask application
    config = CompilerConfig(
        app_name='MyApp',
        additional_binaries={
            '../env/Lib/site-packages/tls_client/dependencies/tls-client-64.dll': 'tls_client/dependencies'
        },
        additional_trees={'../app': 'app'},
    )

    compiler = PythonCompiler(config)
    compiler.compile()

Return