diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 4270216d77..bd8b065955 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -21,4 +21,4 @@ } }, "remoteUser": "root" -} \ No newline at end of file +} diff --git a/.gitattributes b/.gitattributes index 2125666142..1d41792183 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,19 @@ -* text=auto \ No newline at end of file +* text=auto + +# Explicitly declare text files you want to always be normalized and converted to LF upon checkout. +*.c text eol=lf +*.cpp text eol=lf +*.h text eol=lf +*.txt text eol=lf +*.json text eol=lf +*.yml text eol=lf +*.md text eol=lf + +# Explicitly declare text files you want to always be normalized and converted to CRLF upon checkout. +*.cmd text eol=crlf +*.csrpoj text eol=crlf + + +# Denote all files that are truly binary and should not be modified. +*.png binary +*.jpg binary diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 62ee2978f3..83095427dc 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -7,4 +7,4 @@ - [ ] I have read the [contribution guidelines](https://github.com/Azure/azure-osconfig/blob/main/CONTRIBUTING.md). - [ ] I added unit-tests to validate my changes. All unit tests are passing. - [ ] I have merged the latest `main` branch prior to this PR submission. -- [ ] I submitted this PR against the `main` branch. \ No newline at end of file +- [ ] I submitted this PR against the `main` branch. diff --git a/.github/actions/check-run/action.yml b/.github/actions/check-run/action.yml index f8fc90614f..f89441597d 100644 --- a/.github/actions/check-run/action.yml +++ b/.github/actions/check-run/action.yml @@ -56,4 +56,4 @@ runs: -H "Accept: application/vnd.github.v3+json" \ -H "Authorization: Bearer $token" \ -d @data.json - shell: bash \ No newline at end of file + shell: bash diff --git a/.github/actions/container-run/action.yml b/.github/actions/container-run/action.yml index f36f395332..fadfe15d95 100644 --- a/.github/actions/container-run/action.yml +++ b/.github/actions/container-run/action.yml @@ -45,4 +45,4 @@ runs: echo Container ID: $id echo id=$id >> $GITHUB_OUTPUT - shell: bash \ No newline at end of file + shell: bash diff --git a/.github/actions/footprint-check/action.yml b/.github/actions/footprint-check/action.yml index 7ef0a9e86c..cdb561553c 100644 --- a/.github/actions/footprint-check/action.yml +++ b/.github/actions/footprint-check/action.yml @@ -70,4 +70,4 @@ runs: if [ "$result" == 1 ]; then echo "::error Binary footprint validation failed. Delta: $delta" && exit 1 fi - shell: bash \ No newline at end of file + shell: bash diff --git a/.github/actions/gtest-xml/action.yml b/.github/actions/gtest-xml/action.yml index 537dab251f..66c9f0449d 100644 --- a/.github/actions/gtest-xml/action.yml +++ b/.github/actions/gtest-xml/action.yml @@ -68,4 +68,4 @@ runs: with open(output, 'w') as f: new_tree.write(f, encoding='unicode') - shell: python \ No newline at end of file + shell: python diff --git a/.github/actions/pmc/Dockerfile b/.github/actions/pmc/Dockerfile index 753b8fc1fc..32ae08a7f2 100644 --- a/.github/actions/pmc/Dockerfile +++ b/.github/actions/pmc/Dockerfile @@ -2,4 +2,4 @@ FROM mcr.microsoft.com/pmc/pmc-cli:latest COPY entrypoint.sh /entrypoint.sh -ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file +ENTRYPOINT ["/entrypoint.sh"] diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml index cac40edbb6..2404cb0544 100644 --- a/.github/workflows/build-containers.yml +++ b/.github/workflows/build-containers.yml @@ -18,7 +18,7 @@ on: description: 'Rebuild all containers' default: false type: boolean - + env: REGISTRY: ghcr.io BUILD_TYPE: Release diff --git a/.github/workflows/ci-sanitizers.yml b/.github/workflows/ci-sanitizers.yml index dffdaeda85..13b804d9a7 100644 --- a/.github/workflows/ci-sanitizers.yml +++ b/.github/workflows/ci-sanitizers.yml @@ -39,7 +39,7 @@ jobs: fail-fast: false matrix: target: ${{ fromJson(needs.create-ci-matrix.outputs.matrix) }} - + steps: - uses: actions/checkout@v4 with: @@ -118,4 +118,4 @@ jobs: - name: Publish Test Results uses: EnricoMi/publish-unit-test-result-action@v2 with: - files: 'ci-test/*-sanitizer.xml' \ No newline at end of file + files: 'ci-test/*-sanitizer.xml' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1db96e3a9c..5799893f98 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,7 +54,7 @@ jobs: # ] # EOL # )" - # + # # Add arm64 distros only on scheduled runs to prevent long CI times for PRs # if [ "${{ github.event_name }}" == "schedule" ]; then # matrix=$(jq --argjson arm64Targets "$arm64Targets" '. += $arm64Targets' <<< "$matrix") @@ -71,7 +71,7 @@ jobs: fail-fast: false matrix: target: ${{ fromJson(needs.create-ci-matrix.outputs.matrix) }} - + steps: - uses: actions/checkout@v4 with: @@ -149,9 +149,9 @@ jobs: with: path: ci-test pattern: 'unit-test-*' - merge-multiple: true + merge-multiple: true - name: Publish Test Results uses: EnricoMi/publish-unit-test-result-action@v2 with: - files: 'ci-test/*.xml' \ No newline at end of file + files: 'ci-test/*.xml' diff --git a/.github/workflows/formatting.yml b/.github/workflows/formatting.yml new file mode 100644 index 0000000000..e1403d2446 --- /dev/null +++ b/.github/workflows/formatting.yml @@ -0,0 +1,18 @@ +name: Code Formatting + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install pre-commit + run: pip install pre-commit + - name: Run pre-commit + run: pre-commit run --all-files diff --git a/.github/workflows/fuzzing.yml b/.github/workflows/fuzzing.yml index e2ed235f16..4dbdb5403e 100644 --- a/.github/workflows/fuzzing.yml +++ b/.github/workflows/fuzzing.yml @@ -19,4 +19,4 @@ jobs: target: ${{ matrix.target.os }}-${{ matrix.target.version }} arch: ${{ matrix.arch }} package-type: ${{ matrix.target.package-type }} - timeout-seconds: 3600 \ No newline at end of file + timeout-seconds: 3600 diff --git a/.github/workflows/insiders-fast.yml b/.github/workflows/insiders-fast.yml index 30430ada8c..9e3f76e77a 100644 --- a/.github/workflows/insiders-fast.yml +++ b/.github/workflows/insiders-fast.yml @@ -46,7 +46,7 @@ jobs: matrix: target: [ - # Skip ARM64 CI builds for now + # Skip ARM64 CI builds for now { os: amazonlinux, version: 2, arch: amd64, dist: amazonlinux2, package-type: RPM, repo: microsoft-amazonlinux2-insiders-fast-prod-yum, container-tag: latest }, { os: centos, version: 7, arch: amd64, dist: centos7, package-type: RPM, repo: microsoft-centos7-insiders-fast-prod-yum, container-tag: latest }, { os: centos, version: 8, arch: amd64, dist: centos8, package-type: RPM, repo: microsoft-centos8-insiders-fast-prod-yum, container-tag: latest }, diff --git a/.github/workflows/module-test.yml b/.github/workflows/module-test.yml index 80eb9bc924..c4e50aac78 100644 --- a/.github/workflows/module-test.yml +++ b/.github/workflows/module-test.yml @@ -32,4 +32,4 @@ jobs: target: ${{ matrix.target.os }}-${{ matrix.target.version }} arch: ${{ matrix.arch }} package-type: ${{ matrix.target.package-type }} - excluded-tests: ${{ matrix.target.excluded-tests }} \ No newline at end of file + excluded-tests: ${{ matrix.target.excluded-tests }} diff --git a/.github/workflows/prod.yml b/.github/workflows/prod.yml index 738736d9e4..b4d0593697 100644 --- a/.github/workflows/prod.yml +++ b/.github/workflows/prod.yml @@ -65,4 +65,4 @@ jobs: --target ${{ github.sha }} \ --generate-notes \ --latest \ - *.deb \ No newline at end of file + *.deb diff --git a/.github/workflows/universalnrp-test.yml b/.github/workflows/universalnrp-test.yml index 162029cf3b..065577accd 100644 --- a/.github/workflows/universalnrp-test.yml +++ b/.github/workflows/universalnrp-test.yml @@ -152,4 +152,4 @@ jobs: - name: Publish Test Results uses: EnricoMi/publish-unit-test-result-action@v2 with: - files: 'universal-nrp-test/*testResults.xml' \ No newline at end of file + files: 'universal-nrp-test/*testResults.xml' diff --git a/.gitignore b/.gitignore index e19d8d5441..33058f96c3 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,4 @@ node_modules # Terraform .terraform/ .terraform.lock.hcl -*.tfstate* \ No newline at end of file +*.tfstate* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..a057bcf605 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,34 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: end-of-file-fixer + exclude: | + (?x)( + ^src/tests/fuzzer/seed_corpus/.*$ + )$ + - id: trailing-whitespace + - id: mixed-line-ending + args: ['--fix', 'lf'] + files: | + (?x)( + .*\.c| + .*\.h| + .*\.cpp| + .*\.hpp| + .*\.cs| + .*\.md| + .*\.cmake| + .*\.toml| + .*\.json| + .*\.txt + )$ + - id: mixed-line-ending + args: ['--fix', 'crlf'] + files: | + (?x)( + .*\.cmd| + .*\.csrpoj + )$ diff --git a/.vscode/settings.json b/.vscode/settings.json index da10624925..595b5bd700 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -13,4 +13,4 @@ "files.associations": { "*.json": "jsonc" } -} \ No newline at end of file +} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0870cf1d47..e9f2bc291a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,4 +27,4 @@ The OSConfig Core team will respond to a PR that passes all checks in 3 business # Contact -You may contact the OSConfig Core team at [osconfigcore@microsoft.com](mailto:osconfigcore@microsoft.com) to ask questions about OSConfig, to report bugs, to suggest new features, or inquire about any other OSConfig related topic. \ No newline at end of file +You may contact the OSConfig Core team at [osconfigcore@microsoft.com](mailto:osconfigcore@microsoft.com) to ask questions about OSConfig, to report bugs, to suggest new features, or inquire about any other OSConfig related topic. diff --git a/LICENSE.md b/LICENSE.md index 7c4b1bb2d6..c98b6de74f 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -20,4 +20,4 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -``` \ No newline at end of file +``` diff --git a/README.md b/README.md index 1bdd6941e2..c31226fa80 100644 --- a/README.md +++ b/README.md @@ -1,298 +1,298 @@ -# OSConfig - -[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE.md) - -Azure OSConfig is a modular security configuration stack for Linux Edge devices. OSConfig supports multi-authority device management over Azure and Azure Portal/CLI, GitOps, as well as local management. - -For more information on OSConfig see [OSConfig North Star Architecture](docs/architecture.md) and [OSConfig Management Modules](docs/modules.md). - -For our code of conduct and contributing instructions see [CONTRIBUTING](CONTRIBUTING.md). For our approach to security see [SECURITY](SECURITY.md). - -For the list of our published binary packages and instructions how to install them see [devops/README.md](devops/README.md). - -### C Standard - -OSConfig's C/C++ code currently targets compliance with C11. - -## Getting started - -### Prerequisites - -Make sure all dependencies are installed. For example, on Ubuntu: - -```bash -sudo apt-get update -sudo apt-get install -y git cmake build-essential curl libcurl4-openssl-dev libssl-dev uuid-dev libgtest-dev libgmock-dev rapidjson-dev -``` - -Verify that CMake is at least version 3.2.0 and gcc is at least version 4.4.7. - -```bash -cmake --version -gcc --version -``` - -For IoT Hub management, you can install and configure the *Azure IoT Identity Service (AIS)* package as described at [azure.github.io/iot-identity-service/](https://azure.github.io/iot-identity-service/). - -### Build - -Create a folder build folder under the repo root /build - -```bash -mkdir build && cd build -``` - -Build with the following commands issued from under the build subfolder: - -```bash -cmake ../src -DCMAKE_BUILD_TYPE=Release|Debug -Duse_prov_client=ON -Dhsm_type_symm_key=ON -DBUILD_TESTS=ON|OFF -cmake --build . --config Release|Debug --target install -``` -The following OSConfig files are binplaced at build time: - -Source | Destination | Description ------|-----|----- -[src/adapters/pnp/](src/adapters/pnp/) | /usr/bin/osconfig | The OSConfig Agent and the main control binary for OSConfig -[src/platform/](src/platform/) | /usr/bin/osconfig-platform | The OSConfig Platform binary -[src/adapters/pnp/daemon/osconfig.conn](src/adapters/pnp/daemon/osconfig.conn) | /etc/osconfig/osconfig.conn | Holds manual IoT Hub device connection id string (optional) -[src/adapters/pnp/daemon/osconfig.json](src/adapters/pnp/daemon/osconfig.json) | /etc/osconfig/osconfig.json | The main configuration file for OSConfig -[src/modules/commandrunner/assets/osconfig_commandrunner.cache](src/modules/commandrunner/assets/osconfig_commandrunner.cache) | /etc/osconfig/osconfig_commandrunner.cache | Persistent cache for the CommandRunner module -[src/adapters/pnp/daemon/osconfig.service](src/adapters/pnp/daemon/osconfig.service) | /etc/systemd/system/osconfig.service | The service unit for the OSConfig Agent -[src/platform/daemon/osconfig-platform.service](src/platform/daemon/osconfig-platform.service) | /etc/systemd/system/osconfig-platform.service | The service unit for the OSConfig Platform -[src/adapters/pnp/daemon/osconfig.toml](src/adapters/pnp/daemon/osconfig.toml) | /etc/aziot/identityd/config.d/osconfig.toml | The OSConfig Module configuration for AIS -[src/modules/deviceinfo/](src/modules/deviceinfo/) | /usr/lib/osconfig/deviceinfo.so | The DeviceInfo module binary -[src/modules/commandrunner/](src/modules/commandrunner/) | /usr/lib/osconfig/commandrunner.so | The CommandRunner module binary -[src/modules/configuration/](src/modules/configuration/) | /usr/lib/osconfig/configuration.so | The Configuration module binary -[src/modules/securitybaseline/](src/modules/securitybaseline/) | /usr/lib/osconfig/securitybaseline.so | The SecurityBaseline module binary - -### Enable and start OSConfig for the first time - -Enable and start OSConfig for the first time by enabling and starting the OSConfig Agent Daemon (`osconfig`): - -```bash -sudo systemctl daemon-reload -sudo systemctl enable osconfig -sudo systemctl start osconfig -``` - -The OSConfig Agent service is configured to be allowed to be restarted (automatically by systemd or manually by user) for a maximum number of 3 times at 5 minutes intervals. There is a total delay of 16 minutes before the OSConfig Agent service could be restarted again by the user unless the user reboots the device. - -The OSCOnfig Management Platform Daemon (`osconfig-platform`) is automatically started and stopped by the OSConfig Agent service (`osconfig`) but also can be manually started and stopped separately by itself. - -Other daemon control operations: - -```bash -sudo systemctl status osconfig | osconfig-platform -sudo systemctl disable osconfig | osconfig-platform -sudo systemctl stop osconfig | osconfig-platform -``` -To replace a service binary while OSConfig is running: stop the Agent daemon, rebuild, start the Agent daemon. -To replace a service unit while the daemon is running: stop the Agent daemon, disable the Agent amnd Platform daemons, rebuild, reload daemons, start and enable the Agent daemon. - -## Logs - -OSConfig logs to its own logs at `/var/log/osconfig*.log*`: - -```bash -sudo cat /var/log/osconfig_pnp_agent.log -sudo cat /var/log/osconfig_platform.log -sudo cat /var/log/osconfig_commandrunner.log -sudo cat /var/log/osconfig_networking.log -sudo cat /var/log/osconfig_firewall.log -sudo cat /var/log/osconfig_tpm.log -... -``` - -Each of these log files when it reaches maximum size (128 KB) gets rolled over to a file with the same name and a .bak extension (osconfig_agent.bak, for example). - -When OSConfig exists prematurely (crashes) the Agent's log (osconfig_pnp_agent.log) at the very end may contain an indication of that. For example: - -``` -[ERROR] OSConfig crash due to segmentation fault (SIGSEGV) during MpiGet to Firewall.FirewallRules -``` -Only the root user can view these log files. - -## Configuration - -OSConfig can be configured via `/etc/osconfig/osconfig.json`. After changing this configuration file, restart OSConfig to apply the configuration changes. Only the root user can view or edit this configuration file. - -### Enabling management via IoT Hub - -Originally OSConfig was developed with the IoT Hub management channel by default and always enabled. Currently, this managament channel is by default disabled. You can enable it via the OSConfig general configuration file at `/etc/osconfig/osconfig.json`. Edit there the integer value named "IotHubManagement" to a non-zero value: - -```json -{ - "IotHubManagement": 0 -} -``` - -### Adjusting the reporting interval - -OSConfig periodically reports device data at a default time period of 30 seconds. This interval period can be adjusted between 1 second and 86,400 seconds (24 hours) via the OSConfig general configuration file at `/etc/osconfig/osconfig.json`. Edit there the integer value named "ReportingIntervalSeconds" to a value between 1 and 86400: - -```json -{ - "ReportingIntervalSeconds": 30 -} -``` - -This interval is used for RC/DC, GitOps DC, and IoT Hub processing. - -### Enabling logging of system commands executed by OSConfig for debugging purposes - -Command logging means that OSConfig will log all input and output from system commands executed by Agent, Platform and Modules. - -Generally it is not recommended to run OSConfig with command logging enabled. - -To enable command logging for debugging purposes, edit the OSConfig general configuration file `/etc/osconfig/osconfig.json` and set there (or add if needed) an integer value named "CommandLogging" to a non-zero value: - -```json -{ - "CommandLogging": 1 -} -``` - -To disable command logging, set "CommandLogging" to 0. - -### Enabling full logging for debugging purposes - -Full logging means that OSConfig will log all input and output from and to IoT Hub, AIS, RC/DC, GitOps DC, etc. - -Generally it is not recommended to run OSConfig with full logging enabled. - -To enable full logging for debugging purposes, edit the OSConfig general configuration file `/etc/osconfig/osconfig.json` and set there (or add if needed) an integer value named "FullLogging" to a non-zero value: - -```json -{ - "FullLogging": 1 -} -``` - -To disable full logging, set "FullLogging" to 0. - -## Local Management over RC/DC - -OSConfig uses two local files as local digital twins in MIM JSON payload format: - -`/etc/osconfig/osconfig_desired.json` contains desired configuration (to be applied to the device) - -`/etc/osconfig/osconfig_reported.json` contains reported configuration (to be reported from the device) - -This pair of files are called Reported Configuration (RC) and Desired Configuration (DC) or RC/DC. - -Once created, only the root user can view these files or change the DC file. - -By default, the reported configuration is not saved locally to the DC file at `/etc/osconfig/osconfig_reported.json` (local reporting is disabled) and desired configuration is not picked-up from the DC file at `/etc/osconfig/osconfig_desired.json`. - -To enable local management, edit the OSConfig general configuration file `/etc/osconfig/osconfig.json` and set there (or add if needed) an integer value named "LocalManagement" to a non-zero value: - -```json -{ - "LocalManagement": 1 -} -``` -To disable local management, set "LocalManagement" to 0. - -### Desired Configuration (DC) management over GitOps - -OSConfig can apply to the device desired configuration in MIM JSON payload format (same as for RC/DC) read from a Git repository and branch. The DC file must be named `osconfig_desired.json` and be placed in the root of the repository. - -By default, desired configuration (DC) over GitOps is disabled and there are no configured Git repository or branch. - -To enable GitOps DC management, edit the OSConfig general configuration file `/etc/osconfig/osconfig.json` and there: - -1. Set (or add if needed) a string value named "GitRepositoryUrl" to a string value containing the string that can be used to clone a Git repository, for example (this example uses OSConfig's own repository but can be anything): - -```json -{ - "GitRepositoryUrl": "https://github.com/Azure/azure-osconfig" -} -``` - -For HTTPS cloning of a private Git repository, add necessary credentials to the "GitRepositoryUrl" such as, for example: `https://:@github.com/path/to/repo`. For SSH cloning, configure authetication separately on the device so OSConfig can use it. - -2. Set (or add if needed) a string value named "GitBranch" to a string value containing the Git branch name where the DC file is located, for example: - -```json -{ - "GitBranch": "name/branch" -} -``` - -Set (or add if needed) an integer value named "GitManagement" to a non-zero value to enable GitOps DC management: - -```json -{ - "GitManagement": 1 -} -``` -To disable GitOps DC management, set "GitManagement" to 0. - -OSConfig clones locally the configured Git DC file and branch to `/etc/osconfig/gitops/osconfig_desired.json`. This Git clone is automatically deleted when the OSConfig Agent (Watcher) terminates. While active, the cloned DC file is protected for root user access only. - -### Changing the protocol OSConfig uses to connect to the IoT Hub - -The networking protocol that OSConfig uses to connect to the IoT Hub is configured in the OSConfig general configuration file `/etc/osconfig/osconfig.json`: - -```json -{ - "IotHubProtocol": 2 -} -``` - -OSConfig currently supports the following protocol values: - -Value | Description ------|----- -0 | Decided by OSConfig (currently this is MQTT) -1 | MQTT -2 | MQTT over Web Socket - -## HTTP proxy configuration - -When the configured IotHubProtocol value is set to value 2 (MQTT over Web Socket) OSConfig attempts to use the HTTP proxy information configured in one of the following environment variables, the first such variable that is locally present: - -1. `http_proxy` -1. `https_proxy` -1. `HTTP_PROXY` -1. `HTTPS_PROXY` - -OSConfig supports the HTTP proxy configuration to be in one of the following formats: - -- `http://server:port` -- `http://username:password@server:port` -- `http://domain\username:password@server:port` - -Where the prefix is either lowercase `http` or uppercase `HTTP` and the username and password can contain `@` characters escaped as `\@`. - -For example: `http://username\@mail.foo:p\@ssw\@rd@www.foo.org:100` where username is `username@mail.foo`, password is `p@ssw@rd`, the proxy server is `www.foo.org` and the port is 100. - -The environment variable needs to be set for the root user account. For example, for a fictive proxy server, user and password, the environment variable `http_proxy` can be set for the root user manually via console with: - -``` -sudo -E su -export http_proxy=http://user:password@wwww.foo.org:100// -``` - -The environment variable can also be set in the OSConfig service unit file by uncommenting and editing the following line in [src/adapters/pnp/daemon/osconfig.service](src/adapters/pnp/daemon/osconfig.service): - -``` -# Uncomment and edit next line to configure OSConfig with a proxy to connect to the IoT Hub -# Environment="http_proxy=http://user:password@wwww.foo.org:100//" -``` - -After editing the service unit file, stop and disable osconfig.service, rebuild OSConfig, then enable and start osconfig.service: - -```bash -sudo systemctl stop osconfig.service -sudo systemctl disable osconfig.service -cd build -cmake ../src -DCMAKE_BUILD_TYPE=Release|Debug -Duse_prov_client=ON -Dhsm_type_symm_key=ON -DBUILD_TESTS=ON|OFF -cmake --build . --config Release|Debug --target install -sudo systemctl enable osconfig.service -sudo systemctl start osconfig.service -``` - ---- - -Microsoft may collect performance and usage information which may be used to provide and improve Microsoft products and services and enhance users experience. To learn more, review the [privacy statement](https://go.microsoft.com/fwlink/?LinkId=521839&clcid=0x409). \ No newline at end of file +# OSConfig + +[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE.md) + +Azure OSConfig is a modular security configuration stack for Linux Edge devices. OSConfig supports multi-authority device management over Azure and Azure Portal/CLI, GitOps, as well as local management. + +For more information on OSConfig see [OSConfig North Star Architecture](docs/architecture.md) and [OSConfig Management Modules](docs/modules.md). + +For our code of conduct and contributing instructions see [CONTRIBUTING](CONTRIBUTING.md). For our approach to security see [SECURITY](SECURITY.md). + +For the list of our published binary packages and instructions how to install them see [devops/README.md](devops/README.md). + +### C Standard + +OSConfig's C/C++ code currently targets compliance with C11. + +## Getting started + +### Prerequisites + +Make sure all dependencies are installed. For example, on Ubuntu: + +```bash +sudo apt-get update +sudo apt-get install -y git cmake build-essential curl libcurl4-openssl-dev libssl-dev uuid-dev libgtest-dev libgmock-dev rapidjson-dev +``` + +Verify that CMake is at least version 3.2.0 and gcc is at least version 4.4.7. + +```bash +cmake --version +gcc --version +``` + +For IoT Hub management, you can install and configure the *Azure IoT Identity Service (AIS)* package as described at [azure.github.io/iot-identity-service/](https://azure.github.io/iot-identity-service/). + +### Build + +Create a folder build folder under the repo root /build + +```bash +mkdir build && cd build +``` + +Build with the following commands issued from under the build subfolder: + +```bash +cmake ../src -DCMAKE_BUILD_TYPE=Release|Debug -Duse_prov_client=ON -Dhsm_type_symm_key=ON -DBUILD_TESTS=ON|OFF +cmake --build . --config Release|Debug --target install +``` +The following OSConfig files are binplaced at build time: + +Source | Destination | Description +-----|-----|----- +[src/adapters/pnp/](src/adapters/pnp/) | /usr/bin/osconfig | The OSConfig Agent and the main control binary for OSConfig +[src/platform/](src/platform/) | /usr/bin/osconfig-platform | The OSConfig Platform binary +[src/adapters/pnp/daemon/osconfig.conn](src/adapters/pnp/daemon/osconfig.conn) | /etc/osconfig/osconfig.conn | Holds manual IoT Hub device connection id string (optional) +[src/adapters/pnp/daemon/osconfig.json](src/adapters/pnp/daemon/osconfig.json) | /etc/osconfig/osconfig.json | The main configuration file for OSConfig +[src/modules/commandrunner/assets/osconfig_commandrunner.cache](src/modules/commandrunner/assets/osconfig_commandrunner.cache) | /etc/osconfig/osconfig_commandrunner.cache | Persistent cache for the CommandRunner module +[src/adapters/pnp/daemon/osconfig.service](src/adapters/pnp/daemon/osconfig.service) | /etc/systemd/system/osconfig.service | The service unit for the OSConfig Agent +[src/platform/daemon/osconfig-platform.service](src/platform/daemon/osconfig-platform.service) | /etc/systemd/system/osconfig-platform.service | The service unit for the OSConfig Platform +[src/adapters/pnp/daemon/osconfig.toml](src/adapters/pnp/daemon/osconfig.toml) | /etc/aziot/identityd/config.d/osconfig.toml | The OSConfig Module configuration for AIS +[src/modules/deviceinfo/](src/modules/deviceinfo/) | /usr/lib/osconfig/deviceinfo.so | The DeviceInfo module binary +[src/modules/commandrunner/](src/modules/commandrunner/) | /usr/lib/osconfig/commandrunner.so | The CommandRunner module binary +[src/modules/configuration/](src/modules/configuration/) | /usr/lib/osconfig/configuration.so | The Configuration module binary +[src/modules/securitybaseline/](src/modules/securitybaseline/) | /usr/lib/osconfig/securitybaseline.so | The SecurityBaseline module binary + +### Enable and start OSConfig for the first time + +Enable and start OSConfig for the first time by enabling and starting the OSConfig Agent Daemon (`osconfig`): + +```bash +sudo systemctl daemon-reload +sudo systemctl enable osconfig +sudo systemctl start osconfig +``` + +The OSConfig Agent service is configured to be allowed to be restarted (automatically by systemd or manually by user) for a maximum number of 3 times at 5 minutes intervals. There is a total delay of 16 minutes before the OSConfig Agent service could be restarted again by the user unless the user reboots the device. + +The OSCOnfig Management Platform Daemon (`osconfig-platform`) is automatically started and stopped by the OSConfig Agent service (`osconfig`) but also can be manually started and stopped separately by itself. + +Other daemon control operations: + +```bash +sudo systemctl status osconfig | osconfig-platform +sudo systemctl disable osconfig | osconfig-platform +sudo systemctl stop osconfig | osconfig-platform +``` +To replace a service binary while OSConfig is running: stop the Agent daemon, rebuild, start the Agent daemon. +To replace a service unit while the daemon is running: stop the Agent daemon, disable the Agent amnd Platform daemons, rebuild, reload daemons, start and enable the Agent daemon. + +## Logs + +OSConfig logs to its own logs at `/var/log/osconfig*.log*`: + +```bash +sudo cat /var/log/osconfig_pnp_agent.log +sudo cat /var/log/osconfig_platform.log +sudo cat /var/log/osconfig_commandrunner.log +sudo cat /var/log/osconfig_networking.log +sudo cat /var/log/osconfig_firewall.log +sudo cat /var/log/osconfig_tpm.log +... +``` + +Each of these log files when it reaches maximum size (128 KB) gets rolled over to a file with the same name and a .bak extension (osconfig_agent.bak, for example). + +When OSConfig exists prematurely (crashes) the Agent's log (osconfig_pnp_agent.log) at the very end may contain an indication of that. For example: + +``` +[ERROR] OSConfig crash due to segmentation fault (SIGSEGV) during MpiGet to Firewall.FirewallRules +``` +Only the root user can view these log files. + +## Configuration + +OSConfig can be configured via `/etc/osconfig/osconfig.json`. After changing this configuration file, restart OSConfig to apply the configuration changes. Only the root user can view or edit this configuration file. + +### Enabling management via IoT Hub + +Originally OSConfig was developed with the IoT Hub management channel by default and always enabled. Currently, this managament channel is by default disabled. You can enable it via the OSConfig general configuration file at `/etc/osconfig/osconfig.json`. Edit there the integer value named "IotHubManagement" to a non-zero value: + +```json +{ + "IotHubManagement": 0 +} +``` + +### Adjusting the reporting interval + +OSConfig periodically reports device data at a default time period of 30 seconds. This interval period can be adjusted between 1 second and 86,400 seconds (24 hours) via the OSConfig general configuration file at `/etc/osconfig/osconfig.json`. Edit there the integer value named "ReportingIntervalSeconds" to a value between 1 and 86400: + +```json +{ + "ReportingIntervalSeconds": 30 +} +``` + +This interval is used for RC/DC, GitOps DC, and IoT Hub processing. + +### Enabling logging of system commands executed by OSConfig for debugging purposes + +Command logging means that OSConfig will log all input and output from system commands executed by Agent, Platform and Modules. + +Generally it is not recommended to run OSConfig with command logging enabled. + +To enable command logging for debugging purposes, edit the OSConfig general configuration file `/etc/osconfig/osconfig.json` and set there (or add if needed) an integer value named "CommandLogging" to a non-zero value: + +```json +{ + "CommandLogging": 1 +} +``` + +To disable command logging, set "CommandLogging" to 0. + +### Enabling full logging for debugging purposes + +Full logging means that OSConfig will log all input and output from and to IoT Hub, AIS, RC/DC, GitOps DC, etc. + +Generally it is not recommended to run OSConfig with full logging enabled. + +To enable full logging for debugging purposes, edit the OSConfig general configuration file `/etc/osconfig/osconfig.json` and set there (or add if needed) an integer value named "FullLogging" to a non-zero value: + +```json +{ + "FullLogging": 1 +} +``` + +To disable full logging, set "FullLogging" to 0. + +## Local Management over RC/DC + +OSConfig uses two local files as local digital twins in MIM JSON payload format: + +`/etc/osconfig/osconfig_desired.json` contains desired configuration (to be applied to the device) + +`/etc/osconfig/osconfig_reported.json` contains reported configuration (to be reported from the device) + +This pair of files are called Reported Configuration (RC) and Desired Configuration (DC) or RC/DC. + +Once created, only the root user can view these files or change the DC file. + +By default, the reported configuration is not saved locally to the DC file at `/etc/osconfig/osconfig_reported.json` (local reporting is disabled) and desired configuration is not picked-up from the DC file at `/etc/osconfig/osconfig_desired.json`. + +To enable local management, edit the OSConfig general configuration file `/etc/osconfig/osconfig.json` and set there (or add if needed) an integer value named "LocalManagement" to a non-zero value: + +```json +{ + "LocalManagement": 1 +} +``` +To disable local management, set "LocalManagement" to 0. + +### Desired Configuration (DC) management over GitOps + +OSConfig can apply to the device desired configuration in MIM JSON payload format (same as for RC/DC) read from a Git repository and branch. The DC file must be named `osconfig_desired.json` and be placed in the root of the repository. + +By default, desired configuration (DC) over GitOps is disabled and there are no configured Git repository or branch. + +To enable GitOps DC management, edit the OSConfig general configuration file `/etc/osconfig/osconfig.json` and there: + +1. Set (or add if needed) a string value named "GitRepositoryUrl" to a string value containing the string that can be used to clone a Git repository, for example (this example uses OSConfig's own repository but can be anything): + +```json +{ + "GitRepositoryUrl": "https://github.com/Azure/azure-osconfig" +} +``` + +For HTTPS cloning of a private Git repository, add necessary credentials to the "GitRepositoryUrl" such as, for example: `https://:@github.com/path/to/repo`. For SSH cloning, configure authetication separately on the device so OSConfig can use it. + +2. Set (or add if needed) a string value named "GitBranch" to a string value containing the Git branch name where the DC file is located, for example: + +```json +{ + "GitBranch": "name/branch" +} +``` + +Set (or add if needed) an integer value named "GitManagement" to a non-zero value to enable GitOps DC management: + +```json +{ + "GitManagement": 1 +} +``` +To disable GitOps DC management, set "GitManagement" to 0. + +OSConfig clones locally the configured Git DC file and branch to `/etc/osconfig/gitops/osconfig_desired.json`. This Git clone is automatically deleted when the OSConfig Agent (Watcher) terminates. While active, the cloned DC file is protected for root user access only. + +### Changing the protocol OSConfig uses to connect to the IoT Hub + +The networking protocol that OSConfig uses to connect to the IoT Hub is configured in the OSConfig general configuration file `/etc/osconfig/osconfig.json`: + +```json +{ + "IotHubProtocol": 2 +} +``` + +OSConfig currently supports the following protocol values: + +Value | Description +-----|----- +0 | Decided by OSConfig (currently this is MQTT) +1 | MQTT +2 | MQTT over Web Socket + +## HTTP proxy configuration + +When the configured IotHubProtocol value is set to value 2 (MQTT over Web Socket) OSConfig attempts to use the HTTP proxy information configured in one of the following environment variables, the first such variable that is locally present: + +1. `http_proxy` +1. `https_proxy` +1. `HTTP_PROXY` +1. `HTTPS_PROXY` + +OSConfig supports the HTTP proxy configuration to be in one of the following formats: + +- `http://server:port` +- `http://username:password@server:port` +- `http://domain\username:password@server:port` + +Where the prefix is either lowercase `http` or uppercase `HTTP` and the username and password can contain `@` characters escaped as `\@`. + +For example: `http://username\@mail.foo:p\@ssw\@rd@www.foo.org:100` where username is `username@mail.foo`, password is `p@ssw@rd`, the proxy server is `www.foo.org` and the port is 100. + +The environment variable needs to be set for the root user account. For example, for a fictive proxy server, user and password, the environment variable `http_proxy` can be set for the root user manually via console with: + +``` +sudo -E su +export http_proxy=http://user:password@wwww.foo.org:100// +``` + +The environment variable can also be set in the OSConfig service unit file by uncommenting and editing the following line in [src/adapters/pnp/daemon/osconfig.service](src/adapters/pnp/daemon/osconfig.service): + +``` +# Uncomment and edit next line to configure OSConfig with a proxy to connect to the IoT Hub +# Environment="http_proxy=http://user:password@wwww.foo.org:100//" +``` + +After editing the service unit file, stop and disable osconfig.service, rebuild OSConfig, then enable and start osconfig.service: + +```bash +sudo systemctl stop osconfig.service +sudo systemctl disable osconfig.service +cd build +cmake ../src -DCMAKE_BUILD_TYPE=Release|Debug -Duse_prov_client=ON -Dhsm_type_symm_key=ON -DBUILD_TESTS=ON|OFF +cmake --build . --config Release|Debug --target install +sudo systemctl enable osconfig.service +sudo systemctl start osconfig.service +``` + +--- + +Microsoft may collect performance and usage information which may be used to provide and improve Microsoft products and services and enhance users experience. To learn more, review the [privacy statement](https://go.microsoft.com/fwlink/?LinkId=521839&clcid=0x409). diff --git a/SECURITY.md b/SECURITY.md index 26f92f8399..bc93355a0b 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -32,4 +32,4 @@ We prefer all communications to be in English. ## Policy -Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://www.microsoft.com/en-us/msrc/cvd). \ No newline at end of file +Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://www.microsoft.com/en-us/msrc/cvd). diff --git a/devops/README.md b/devops/README.md index 5ebf6c5dbd..aeb9c4b853 100644 --- a/devops/README.md +++ b/devops/README.md @@ -8,7 +8,7 @@ Our packages are distributed via the [Microsoft repository](https://packages.microsoft.com/). The following instructions will guide you through the installation of the `osconfig` package on our supported Linux distributions. Additionally, you can find our MSDN documentation on package installation at [docs.microsoft.com](https://learn.microsoft.com/azure/osconfig/howto-install?tabs=package). Channels | Description ------|----- +-----|----- `insiders-fast` | The latest and greatest features and bug fixes, represents current work-in-progress so things might not work as expected. This channel is updated frequently. `prod` | The stable channel. This channel is updated less frequently and is recommended for production use. @@ -272,4 +272,4 @@ This folder contains the artifacts and scripts used by our engineering systems ( - `e2e` - `cloudtest` - backup of 1ES managed CloudTest image definitions - `terraform` - terraform modules for provisioning Azure resources for E2E tests -- `rpm` - RPM packaging scripts \ No newline at end of file +- `rpm` - RPM packaging scripts diff --git a/devops/debian/README.md b/devops/debian/README.md index 44c455ae43..b842d96bdf 100644 --- a/devops/debian/README.md +++ b/devops/debian/README.md @@ -43,7 +43,7 @@ The scripts contained in this directory are automatically generated by debhelper ``` * Copy the `osconfig.service` file into the `debian` directory. The debhelpers will automatically detect a service file and use it to stage the the scripts accordingly. * Modify `debian/rules` and add a new target called `override_dh_installsystemd` which performs the `dh_installsystemd` command like so.... - + ``` override_dh_installsystemd: dh_installsystemd --name=osconfig @@ -98,4 +98,4 @@ override_dh_installsystemd: `-- osconfig-0.3.0.tar.gz ``` * To builds the scripts in this directory, we use the template as a base and replace the `#DEBHELPER#` with the contents of `/osconfig-0.3.0/debian/osconfig/DEBIAN/