diff --git a/.gitignore b/.gitignore index 5ef997d..140ea71 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ api/api api/package-lock.json lxc/i lxc/lockfile +container/build.yaml +container/*.tar.xz diff --git a/container/build.sh b/container/build.sh new file mode 100755 index 0000000..c7f7355 --- /dev/null +++ b/container/build.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +python3 -m pip install pyyaml +python3 configure.py +distrobuilder build-lxc build.yaml + diff --git a/container/configure.py b/container/configure.py new file mode 100644 index 0000000..5c02fd3 --- /dev/null +++ b/container/configure.py @@ -0,0 +1,13 @@ +import yaml + + +with open('piston.yaml') as dbc: + with open('install_script.sh') as install_script_file: + with open('build.yaml' , 'w+') as distrobuilder_config_file_new: + distrobuilder_config = yaml.safe_load(dbc) + distrobuilder_config['actions'].append({ + 'trigger': 'post-packages', + 'action': install_script_file.read(), + + }) + yaml.dump(distrobuilder_config, distrobuilder_config_file_new) diff --git a/container/install_script.sh b/container/install_script.sh new file mode 100644 index 0000000..9631791 --- /dev/null +++ b/container/install_script.sh @@ -0,0 +1,284 @@ +#!/bin/bash +#echo "Don't run this on your system!" && exit 0 + +# install all necessary piston dependencies +echo 'source /opt/.profile' >> /opt/.bashrc +echo 'export HOME=/opt' >> /opt/.profile +echo 'export TERM=linux' >> /opt/.profile +echo 'export PATH=$PATH:/opt/.local/bin' >> /opt/.profile +export HOME=/opt +export TERM=linux +sed -i 's/\/root/\/opt/' /etc/passwd +sed -i \ + 's/http:\/\/archive.ubuntu.com\/ubuntu/http:\/\/mirror.math.princeton.edu\/pub\/ubuntu/' \ + /etc/apt/sources.list +apt-get update +apt-get install -y \ + nano wget build-essential pkg-config libxml2-dev \ + libsqlite3-dev mono-complete curl cmake libpython2.7-dev \ + ruby libtinfo-dev unzip git openssl libssl-dev sbcl libevent-dev + +# install python2 +# final binary: /opt/python2/Python-2.7.17/python +# get version: /opt/python2/Python-2.7.17/python -V +cd /opt && mkdir python2 && cd python2 +wget https://www.python.org/ftp/python/2.7.17/Python-2.7.17.tar.xz +unxz Python-2.7.17.tar.xz +tar -xf Python-2.7.17.tar +cd Python-2.7.17 +./configure +# open Modules/Setup and uncomment zlib line +make +echo 'export PATH=$PATH:/opt/python2/Python-2.7.17' >> /opt/.profile +. /opt/.profile + +# install python3 +# final binary: /opt/python3/Python-3.8.2/python +# get version: /opt/python3/Python-3.8.2/python -V +cd /opt && mkdir python3 && cd python3 +wget https://www.python.org/ftp/python/3.8.2/Python-3.8.2.tar.xz +unxz Python-3.8.2.tar.xz +tar -xf Python-3.8.2.tar +cd Python-3.8.2 +./configure +make +ln -s python python3.8 +echo 'export PATH=$PATH:/opt/python3/Python-3.8.2' >> /opt/.profile +. /opt/.profile + +# install paradoc +# this is not a binary, it is a python module +# therefore it cannot be run directly as it requires python3 to be installed +cd /opt && mkdir paradoc && cd paradoc +git clone https://github.com/betaveros/paradoc.git + +# install node.js +# final binary: /opt/nodejs/node-v12.16.1-linux-x64/bin/node +# get version: /opt/nodejs/node-v12.16.1-linux-x64/bin/node -v +cd /opt && mkdir nodejs && cd nodejs +wget https://nodejs.org/dist/v12.16.1/node-v12.16.1-linux-x64.tar.xz +unxz node-v12.16.1-linux-x64.tar.xz +tar -xf node-v12.16.1-linux-x64.tar +echo 'export PATH=$PATH:/opt/nodejs/node-v12.16.1-linux-x64/bin' >> /opt/.profile +. /opt/.profile + +# install typescript +# final binary: /opt/nodejs/node-v12.16.1-linux-x64/bin/tsc +# get version: /opt/nodejs/node-v12.16.1-linux-x64/bin/tsc -v +/opt/nodejs/node-v12.16.1-linux-x64/bin/npm i -g typescript + +# install golang +# final binary: /opt/go/go/bin/go +# get version: /opt/go/go/bin/go version +cd /opt && mkdir go && cd go +wget https://dl.google.com/go/go1.14.1.linux-amd64.tar.gz +tar -xzf go1.14.1.linux-amd64.tar.gz +echo 'export PATH=$PATH:/opt/go/go/bin' >> /opt/.profile +echo 'export GOROOT=/opt/go/go' >> /opt/.profile +echo 'export GOCACHE=/tmp' >> /opt/.profile +. /opt/.profile + +# install php +# final binary: /usr/local/bin/php +# get version: /usr/local/bin/php -v +cd /opt && mkdir php && cd php +wget https://www.php.net/distributions/php-8.0.0.tar.gz +tar -xzf php-8.0.0.tar.gz +cd php-8.0.0 +./configure +make +make install + +# install rust +# final binary: /usr/local/bin/rustc +# get version: /usr/local/bin/rustc --version +cd /opt && mkdir rust && cd rust +wget https://static.rust-lang.org/dist/rust-1.49.0-x86_64-unknown-linux-gnu.tar.gz +tar -xzf rust-1.49.0-x86_64-unknown-linux-gnu.tar.gz +cd rust-1.49.0-x86_64-unknown-linux-gnu +./install.sh + +# install swift +# final binary: /opt/swift/swift-5.1.5-RELEASE-ubuntu18.04/usr/bin/swift +# get version: /opt/swift/swift-5.1.5-RELEASE-ubuntu18.04/usr/bin/swift --version +cd /opt && mkdir swift && cd swift +wget https://swift.org/builds/swift-5.1.5-release/ubuntu1804/swift-5.1.5-RELEASE/swift-5.1.5-RELEASE-ubuntu18.04.tar.gz +tar -xzf swift-5.1.5-RELEASE-ubuntu18.04.tar.gz +echo 'export PATH=$PATH:/opt/swift/swift-5.1.5-RELEASE-ubuntu18.04/usr/bin' >> /opt/.profile +. /opt/.profile + +# install nasm +# final binary: /opt/nasm/nasm-2.14.02/nasm +# get version: /opt/nasm/nasm-2.14.02/nasm -v +cd /opt && mkdir nasm && cd nasm +wget https://www.nasm.us/pub/nasm/releasebuilds/2.14.02/nasm-2.14.02.tar.gz +tar -xzf nasm-2.14.02.tar.gz +cd nasm-2.14.02 +./configure +make +echo 'export PATH=$PATH:/opt/nasm/nasm-2.14.02' >> /opt/.profile +. /opt/.profile + +# install java +# final binary: /opt/java/jdk-14/bin/java +# get version: /opt/java/jdk-14/bin/java -version +cd /opt && mkdir java && cd java +wget https://download.java.net/java/GA/jdk14/076bab302c7b4508975440c56f6cc26a/36/GPL/openjdk-14_linux-x64_bin.tar.gz +tar -xzf openjdk-14_linux-x64_bin.tar.gz +echo 'export PATH=$PATH:/opt/java/jdk-14/bin' >> /opt/.profile +. /opt/.profile + +# install jelly +cd /opt && mkdir jelly && cd jelly +wget https://github.com/DennisMitchell/jellylanguage/archive/master.zip +unzip master.zip +cd jellylanguage-master +python3.8 -m pip install . +sed -i 's/\/usr\/local\/bin\/python3.8/\/opt\/python3\/Python-3.8.2\/python3.8/' /usr/local/bin/jelly + +# install julia +# final binary: /opt/julia/julia-1.5.0/bin/julia +# get version: /opt/julia/julia-1.5.0/bin/julia --version +cd /opt && mkdir julia && cd julia +wget https://julialang-s3.julialang.org/bin/linux/x64/1.5/julia-1.5.0-linux-x86_64.tar.gz +tar -xzf julia-1.5.0-linux-x86_64.tar.gz +echo 'export PATH=$PATH:/opt/julia/julia-1.5.0/bin' >> /opt/.profile +. /opt/.profile + +# install kotlin +# final binary: /opt/kotlinc/bin/kotlinc +# get version: /opt/kotlinc/bin/kotlinc -version +cd /opt +wget https://github.com/JetBrains/kotlin/releases/download/v1.4.10/kotlin-compiler-1.4.10.zip +unzip kotlin-compiler-1.4.10.zip +rm kotlin-compiler-1.4.10.zip +echo 'export PATH=$PATH:/opt/kotlinc/bin' >> /opt/.profile +. /opt/.profile + +# install elixir and erlang +# final binary: /opt/elixir/bin/elixir +# get version: /opt/elixir/bin/elixir --version +# erlang +cd /opt && mkdir erlang && cd erlang +wget http://erlang.org/download/otp_src_23.0.tar.gz +gunzip -c otp_src_23.0.tar.gz | tar xf - +cd otp_src_23.0 && ./configure +make +echo 'export PATH=$PATH:/opt/erlang/otp_src_23.0/bin' >> /opt/.profile +. /opt/.profile +# elixir +cd /opt && mkdir elixir && cd elixir +wget https://github.com/elixir-lang/elixir/releases/download/v1.10.3/Precompiled.zip +mkdir elixir-1.10.3 && unzip Precompiled.zip -d elixir-1.10.3/ +echo 'export PATH=$PATH:/opt/elixir/elixir-1.10.3/bin' >> /opt/.profile +. /opt/.profile + +# install emacs +# final binary: /opt/emacs/emacs-26.3/src/emacs +# get version: /opt/emacs/emacs-26.3/src/emacs --version +cd /opt && mkdir emacs && cd emacs +wget https://mirrors.ocf.berkeley.edu/gnu/emacs/emacs-26.3.tar.xz +tar -xf emacs-26.3.tar.xz +rm emacs-26.3.tar.xz +cd emacs-26.3 +./configure --with-gnutls=no +make +echo 'export PATH=$PATH:/opt/emacs/emacs-26.3/src' >> /opt/.profile +. /opt/.profile + +# install lua +# final binary: /opt/lua/lua54/src/lua +# get version: /opt/lua/lua54/src/lua -v +cd /opt && mkdir lua && cd lua +wget https://sourceforge.net/projects/luabinaries/files/5.4.0/Docs%20and%20Sources/lua-5.4.0_Sources.tar.gz/download +tar -xzf download +cd lua54 +make +echo 'export PATH=$PATH:/opt/lua/lua54/src' >> /opt/.profile +. /opt/.profile + +# install haskell +# final binary: /usr/bin/ghc +# get version: /usr/bin/ghc --version +apt install -y ghc + +# install deno +# final binary: /opt/.deno/bin/deno +# get version: /opt/.deno/bin/deno --version +cd /opt && mkdir deno && cd deno +curl -fsSL https://deno.land/x/install/install.sh | sh +echo 'export DENO_INSTALL="/opt/.deno"' >> /opt/.profile +echo 'export PATH="$DENO_INSTALL/bin:$PATH"' >> /opt/.profile +. /opt/.profile + +# install brainfuck +cd /opt && mkdir bf && cd bf +git clone https://github.com/texus/Brainfuck-interpreter +cd Brainfuck-interpreter +echo 'export PATH=$PATH:/opt/bf/Brainfuck-interpreter' >> /opt/.profile +. /opt/.profile + +# install crystal +# final binary: /opt/crystal/crystal-0.35.1-1/bin/crystal +# get version: /opt/crystal/crystal-0.35.1-1/bin/crystal -v +cd /opt && mkdir crystal && cd crystal +wget https://github.com/crystal-lang/crystal/releases/download/0.35.1/crystal-0.35.1-1-linux-x86_64.tar.gz +tar -xzf crystal-0.35.1-1-linux-x86_64.tar.gz +echo 'export PATH="$PATH:/opt/crystal/crystal-0.35.1-1/bin:$PATH"' >> /opt/.profile +. /opt/.profile + +# install d +# final binary: /opt/d/dmd2/linux/bin64/dmd +# get version: /opt/d/dmd2/linux/bin64/dmd --version +cd /opt && mkdir d && cd d +wget http://downloads.dlang.org/releases/2.x/2.095.0/dmd.2.095.0.linux.tar.xz +unxz dmd.2.095.0.linux.tar.xz +tar -xf dmd.2.095.0.linux.tar +echo 'export PATH=$PATH:/opt/d/dmd2/linux/bin64' >> /opt/.profile +. /opt/.profile + +# install zig +# final binary: /opt/zig/zig +# get version: /opt/zig/zig version +cd /opt && mkdir zig && cd zig +wget https://ziglang.org/download/0.7.1/zig-linux-x86_64-0.7.1.tar.xz +tar -xf zig-linux-x86_64-0.7.1.tar.xz +mv zig-linux-x86_64-0.7.1 zig +rm zig-linux-x86_64-0.7.1.tar.xz +echo 'export PATH=$PATH:/opt/zig/zig' >> /opt/.profile +. /opt/.profile + +# install nim +# final binary: /opt/nim/bin/nim +# get version: /opt/nim/bin/nim -v +cd /opt && mkdir nim && cd nim +wget https://nim-lang.org/download/nim-1.4.0-linux_x64.tar.xz +unxz nim-1.4.0-linux_x64.tar.xz +tar -xf nim-1.4.0-linux_x64.tar +cd nim-1.4.0 +./install.sh /opt +echo 'export PATH=$PATH:/opt/nim/bin' >> /opt/.profile +. /opt/.profile + +# create runnable users and apply limits +for i in {1..150}; do + useradd -M runner$i + usermod -d /tmp runner$i + echo "runner$i soft nproc 64" >> /etc/security/limits.conf + echo "runner$i hard nproc 64" >> /etc/security/limits.conf + echo "runner$i soft nofile 2048" >> /etc/security/limits.conf + echo "runner$i hard nofile 2048" >> /etc/security/limits.conf +done + +# remove any lingering write access to others +cd /opt +chown -R root: * +chmod -R o-w * + +# cleanup +rm -rf /home/ubuntu +chmod 777 /tmp + +# disable cron +systemctl stop cron +systemctl disable cron diff --git a/container/piston.yaml b/container/piston.yaml new file mode 100644 index 0000000..4cbb9c0 --- /dev/null +++ b/container/piston.yaml @@ -0,0 +1,355 @@ +image: + name: ubuntu-bionic-x86_64-piston + distribution: ubuntu + release: bionic + description: |- + Ubuntu {{ image.release }} preconfigured for Piston + architecture: x86_64 + +source: + downloader: debootstrap + same_as: bionic + url: http://archive.ubuntu.com/ubuntu + keyserver: keyserver.ubuntu.com + keys: + - '0x790BC7277767219C42C86F933B4FE6ACC0B21F32' + - '0xf6ecb3762474eda9d21b7022871920d1991bc93c' + +targets: + lxc: + create-message: |- + You just created an {{ image.description }} container. + To enable SSH, run: apt install openssh-server + No default root or user password are set by LXC. + config: + - type: all + before: 5 + content: |- + lxc.include = LXC_TEMPLATE_CONFIG/ubuntu.common.conf + - type: user + before: 5 + content: |- + lxc.include = LXC_TEMPLATE_CONFIG/ubuntu.userns.conf + - type: all + after: 4 + content: |- + lxc.include = LXC_TEMPLATE_CONFIG/common.conf + # For Ubuntu 14.04 + lxc.mount.entry = /sys/kernel/debug sys/kernel/debug none bind,optional 0 0 + lxc.mount.entry = /sys/kernel/security sys/kernel/security none bind,optional 0 0 + lxc.mount.entry = /sys/fs/pstore sys/fs/pstore none bind,optional 0 0 + lxc.mount.entry = mqueue dev/mqueue mqueue rw,relatime,create=dir,optional 0 0 + - type: user + after: 4 + content: |- + lxc.include = LXC_TEMPLATE_CONFIG/userns.conf + # For Ubuntu 14.04 + lxc.mount.entry = /sys/firmware/efi/efivars sys/firmware/efi/efivars none bind,optional 0 0 + lxc.mount.entry = /proc/sys/fs/binfmt_misc proc/sys/fs/binfmt_misc none bind,optional 0 0 + - type: all + content: |- + lxc.arch = {{ image.architecture_personality }} +files: +- path: /etc/hostname + generator: hostname + +- path: /etc/hosts + generator: hosts + +- path: /etc/resolvconf/resolv.conf.d/original + generator: remove + +- path: /etc/resolvconf/resolv.conf.d/tail + generator: remove + +- path: /etc/machine-id + generator: dump + + +- path: /var/lib/dbus/machine-id + generator: remove + +- path: /etc/netplan/10-lxc.yaml + generator: dump + content: |- + network: + version: 2 + ethernets: + eth0: + dhcp4: true + dhcp-identifier: mac + releases: + - bionic + - eoan + - focal + - groovy + types: + - container + variants: + - default + +- path: /etc/network/interfaces + generator: dump + content: |- + # This file describes the network interfaces available on your system + # and how to activate them. For more information, see interfaces(5). + # The loopback network interface + auto lo + iface lo inet loopback + auto eth0 + iface eth0 inet dhcp + source /etc/network/interfaces.d/*.cfg + releases: + - trusty + - xenial + types: + - container + +- path: /etc/netplan/10-lxc.yaml + generator: dump + content: |- + network: + version: 2 + ethernets: + enp5s0: + dhcp4: true + dhcp-identifier: mac + releases: + - bionic + - eoan + - focal + - groovy + types: + - vm + variants: + - default + +- path: /etc/network/interfaces + generator: dump + content: |- + # This file describes the network interfaces available on your system + # and how to activate them. For more information, see interfaces(5). + # The loopback network interface + auto lo + iface lo inet loopback + auto enp5s0 + iface enp5s0 inet dhcp + source /etc/network/interfaces.d/*.cfg + releases: + - trusty + - xenial + types: + - vm + +- path: /etc/init/lxc-tty.conf + generator: upstart-tty + releases: + - trusty + types: + - container + +- name: meta-data + generator: cloud-init + variants: + - cloud + +- name: network-config + generator: cloud-init + variants: + - cloud + +- name: user-data + generator: cloud-init + variants: + - cloud + +- name: vendor-data + generator: cloud-init + variants: + - cloud + +- name: ext4 + generator: fstab + types: + - vm + +- name: lxd-agent + generator: lxd-agent + types: + - vm + +- path: /etc/default/grub.d/50-lxd.cfg + generator: dump + content: |- + GRUB_RECORDFAIL_TIMEOUT=0 + GRUB_TIMEOUT=0 + GRUB_CMDLINE_LINUX_DEFAULT="${GRUB_CMDLINE_LINUX_DEFAULT} console=tty1 console=ttyS0" + GRUB_TERMINAL=console + types: + - vm + +- path: /etc/sudoers.d/90-lxd + generator: dump + mode: '0440' + content: |- + # User rules for ubuntu + ubuntu ALL=(ALL) NOPASSWD:ALL + variants: + - default + +packages: + manager: apt + update: true + cleanup: true + sets: + - packages: + - apt-transport-https + - fuse + - language-pack-en + - openssh-client + - vim + action: install + + - packages: + - cloud-init + action: install + variants: + - cloud + + - packages: + - acpid + action: install + architectures: + - amd64 + - arm64 + types: + - vm + + - packages: + - grub-efi-amd64-signed + - shim-signed + action: install + architectures: + - amd64 + types: + - vm + + - packages: + - grub-efi-arm64-signed + action: install + architectures: + - arm64 + types: + - vm + + - packages: + - shim-signed + action: install + architectures: + - arm64 + releases: + - disco + - eoan + - focal + - groovy + types: + - vm + + - packages: + - linux-virtual-hwe-16.04 + action: install + releases: + - xenial + types: + - vm + + - packages: + - linux-virtual + action: install + releases: + - bionic + - eoan + - focal + - groovy + types: + - vm + + - packages: + - os-prober + action: remove + types: + - vm + + repositories: + - name: sources.list + url: |- + deb http://archive.ubuntu.com/ubuntu {{ image.release }} main restricted universe multiverse + deb http://archive.ubuntu.com/ubuntu {{ image.release }}-updates main restricted universe multiverse + deb http://security.ubuntu.com/ubuntu {{ image.release }}-security main restricted universe multiverse + architectures: + - amd64 + - i386 + + - name: sources.list + url: |- + deb http://ports.ubuntu.com/ubuntu-ports {{ image.release }} main restricted universe multiverse + deb http://ports.ubuntu.com/ubuntu-ports {{ image.release }}-updates main restricted universe multiverse + deb http://ports.ubuntu.com/ubuntu-ports {{ image.release }}-security main restricted universe multiverse + architectures: + - armhf + - arm64 + - powerpc + - powerpc64 + - ppc64el + +actions: +- trigger: post-update + action: |- + #!/bin/sh + set -eux + # Create the ubuntu user account + getent group sudo >/dev/null 2>&1 || groupadd --system sudo + useradd --create-home -s /bin/bash -G sudo -U ubuntu + variants: + - default + +- trigger: post-packages + action: |- + #!/bin/sh + set -eux + # Enable systemd-networkd + systemctl enable systemd-networkd + releases: + - bionic + - eoan + - focal + - groovy + +- trigger: post-packages + action: |- + #!/bin/sh + set -eux + # Make sure the locale is built and functional + locale-gen en_US.UTF-8 + update-locale LANG=en_US.UTF-8 + # Cleanup underlying /run + mount -o bind / /mnt + rm -rf /mnt/run/* + umount /mnt + # Cleanup temporary shadow paths + rm /etc/*- +- trigger: post-files + action: |- + #!/bin/sh + set -eux + TARGET="x86_64" + [ "$(uname -m)" = "aarch64" ] && TARGET="arm64" + update-grub + grub-install --uefi-secure-boot --target="${TARGET}-efi" --no-nvram --removable + update-grub + sed -i "s#root=[^ ]*#root=/dev/sda2#g" /boot/grub/grub.cfg + types: + - vm + +mappings: + architecture_map: debian diff --git a/container/readme.md b/container/readme.md new file mode 100644 index 0000000..1af19d5 --- /dev/null +++ b/container/readme.md @@ -0,0 +1,7 @@ +# LXC Container Build + +Requires: `lxc`, `lxc-net`, `packer` (Hashicorp Packer) + +To build: `packer build -var 'apt_mirror=[apt mirror]' -var 'make_threads=[-j flag]' piston.pkr.hcl` + +After roughly 30 minutes (on an i7-4790k), you should have an image built \ No newline at end of file diff --git a/readme.md b/readme.md index 24fb0f8..bd96598 100644 --- a/readme.md +++ b/readme.md @@ -141,12 +141,17 @@ cd piston/lxc #### Installation (simple) -- Coming soon. +- Install additional dependencies python3, pip and distrobuilder +- `cd ../container && ./build.sh` +- Wait, it may take up to an hour. +- `lxc-create -n piston -t local -- --metadata meta.tar.xz --fstree rootfs.tar.xz` +- `cd ../lxc && ./start` +- Good to go! + #### Installation (advanced) -- See `var/install.txt` for how to create a new LXC container and install all of the required -software. +- See `var/install.txt` for how to build the container manually #### CLI Usage - `cli/execute [language] [file path] [args]`