-
Notifications
You must be signed in to change notification settings - Fork 49
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[SWDEV-398070] Adding logging to ROCm SMI (by default off)
Updates:
* [rocm-smi] Provide a thread-safe logging feature
* [rocm-smi] Adding logrotation into install/upgrade/remove
scripts
* [rocm-smi] Updated cmake lists to include rocm_smi_logger
* [rocm-smi] Updated DEB/RPM install/remove logging file &
folder with all users having r/w privledges for
/var/log/rocm_smi_lib/ROCm-SMI-lib.log
* [rocm-smi] Added ability to do a glob search for multiple files
(globFileExists), assists doing file searches with * strings
* [rocm-smi] Added ability to log system details when RSMI_LOGGING
is turned on (getSystemDetails())
* [rocm-smi] Added logging to provide which ROCm API is being called
when RSMI_LOGGING is on
* [rocm-smi] Added logging to provide SYSFS path and read value,
when RSMI_LOGGING is on. Provides error reponse on failure.
* [rocm-smi] Added logging to provide SYSFS path and read value,
when RSMI_LOGGING is on. Provides error reponse on failure.
* [rocm-smi] Added environment variable RSMI_LOGGING to control
when logging is enabled or disabled. By default, by not
setting this env. variable, logging is turned off. When
setting RSMI_LOGGING=<any value>, logging is enabled
which is placed in /var/log/rocm_smi_lib/ROCm-SMI-lib.log file.
Setting RSMI_LOGGING is allowed in both debug and release builds.
* [rocm-smi] Removed an initialize procedure which keeps
debug_inf_loop. Seems this feature is not being used.
Change-Id: I79b48387609c6233c6f05b04fb8bba66b68c2399
Signed-off-by: Charis Poag <[email protected]>- Loading branch information
1 parent
ed74bc6
commit c3a095a
Showing
16 changed files
with
1,591 additions
and
13 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,107 @@ | ||
| #!/bin/bash | ||
| #set -x | ||
|
|
||
| do_addLogFolder() { | ||
| sudo mkdir -p /var/log/rocm_smi_lib | ||
| sudo touch /var/log/rocm_smi_lib/ROCm-SMI-lib.log | ||
| sudo chmod -R a+rw /var/log/rocm_smi_lib | ||
| sudo chmod a+rw /var/log/rocm_smi_lib/ROCm-SMI-lib.log | ||
| } | ||
|
|
||
| do_configureLogrotate() { | ||
| logrotate --version &>/dev/null | ||
| if [ $? -ne 0 ]; then | ||
| echo "[WARNING] Detected logrotate is not installed."\ | ||
| "ROCm-smi logs (when turned on) will not rotate properly." | ||
| return | ||
| fi | ||
|
|
||
| if [ ! -f /etc/logrotate.d/rocm_smi.conf ]; then | ||
| sudo touch /etc/logrotate.d/rocm_smi.conf | ||
| sudo chmod 644 /etc/logrotate.d/rocm_smi.conf # root r/w, all others read | ||
| # ROCm SMI logging rotation, rotates files using root user/group | ||
| # Hourly logrotation check | ||
| # Only rotates if size grew larger than 1MB | ||
| # Max of 4 rotation files, oldest will be removed | ||
| # Rotated files use date extention of ex. ROCm-SMI-lib.log.2023-05-09_16:51:42 | ||
| cat <<'EOF' | sudo tee /etc/logrotate.d/rocm_smi.conf >/dev/null | ||
| /var/log/rocm_smi_lib/ROCm-SMI-lib.log { | ||
| su root root | ||
| hourly | ||
| missingok | ||
| notifempty | ||
| rotate 4 | ||
| size 1M | ||
| copytruncate | ||
| dateext | ||
| dateformat .%Y-%m-%d_%H:%M:%S | ||
| } | ||
| EOF | ||
| # workaround: remove extra 'OURCE' text | ||
| # from rocm_smi.conf. Unsure if CMAKE, | ||
| # bash, or here document | ||
| # issue (only seen on RHEL 8.7) | ||
| sudo sed -i s/OURCE//g /etc/logrotate.d/rocm_smi.conf | ||
| fi | ||
| # check if logrotate uses system timers, Ubuntu/modern OS's do | ||
| # Several older OS's like RHEL 8.7, do not. Instead defaults | ||
| # to use daily cron jobs - see https://stackoverflow.com/a/69465677 | ||
| sudo systemctl list-timers|grep -iq logrotate | ||
| if [ $? -ne 0 ]; then | ||
| # confirm logrotate file exists in daily | ||
| if [ -f /etc/cron.daily/logrotate ]; then | ||
| # move logrotate daily to hourly | ||
| if [ -f /etc/cron.hourly/logrotate ]; then | ||
| sudo mv /etc/cron.daily/logrotate /etc/cron.hourly/logrotate | ||
| else | ||
| echo "[WARNING] Could find and configure hourly cron for ROCm-smi's"\ | ||
| " logrotate. ROCm-smi logs (when turned on) will not rotate properly." | ||
| return | ||
| fi | ||
| else | ||
| # confirm that it's already been moved to hourly | ||
| sudo find /etc/cron.* -iname logrotate -print -quit |grep -iq hourly | ||
| if [ $? -ne 0 ]; then | ||
| echo "[WARNING] Could not configure an hourly cron for ROCm-smi's logrotate."\ | ||
| "ROCm-smi logs (when turned on) may not rotate properly." | ||
| fi | ||
| fi | ||
| else | ||
| # Configure systemd timers - the typical setup for modern Linux logrotation setups | ||
| if [ -f /lib/systemd/system/logrotate.timer ]; then | ||
| if [ ! -f /lib/systemd/system/logrotate.timer.backup ]; then | ||
| sudo cp /lib/systemd/system/logrotate.timer /lib/systemd/system/logrotate.timer.backup | ||
| fi | ||
| cat <<'EOF' | sudo tee /lib/systemd/system/logrotate.timer >/dev/null | ||
| [Unit] | ||
| Description=Hourly rotation of log files | ||
| Documentation=man:logrotate(8) man:logrotate.conf(5) | ||
| [Timer] | ||
| OnCalendar= | ||
| OnCalendar=hourly | ||
| AccuracySec=1m | ||
| Persistent=true | ||
| [Install] | ||
| WantedBy=timers.target | ||
| EOF | ||
| sudo systemctl reenable --now logrotate.timer | ||
| else | ||
| echo "[WARNING] Could not configure systemd timer for ROCm's logrotate."\ | ||
| "ROCm-smi logs (when turned on) will not rotate properly." | ||
| fi | ||
| fi | ||
| } | ||
|
|
||
| # left-hand term originates from ENABLE_LDCONFIG = ON/OFF at package build | ||
| if [ "@ENABLE_LDCONFIG@" == "ON" ]; then | ||
| echo -e "@CPACK_PACKAGING_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@" > /etc/ld.so.conf.d/x86_64-librocm_smi_lib.conf | ||
| ldconfig | ||
| fi | ||
|
|
||
| # post install or upgrade, $i is 1 or 2 -> do these actions | ||
| if [ $1 -ge 1 ]; then | ||
| do_addLogFolder | ||
| do_configureLogrotate | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,28 @@ | ||
| #!/bin/bash | ||
| #set -x | ||
|
|
||
| rm_logFolder() { | ||
| sudo rm -rf /var/log/rocm_smi_lib | ||
| } | ||
|
|
||
| return_logrotateToOrigConfig() { | ||
| if [ -f /etc/logrotate.d/rocm_smi.conf ]; then | ||
| sudo rm -rf /etc/logrotate.d/rocm_smi.conf | ||
| fi | ||
| if [ -f /etc/cron.hourly/logrotate ]; then | ||
| sudo mv /etc/cron.hourly/logrotate /etc/cron.daily/logrotate | ||
| fi | ||
| if [ -f /lib/systemd/system/logrotate.timer.backup ]; then | ||
| sudo cp /lib/systemd/system/logrotate.timer.backup /lib/systemd/system/logrotate.timer | ||
| sudo rm -rf /lib/systemd/system/logrotate.timer.backup | ||
| sudo systemctl reenable --now logrotate.timer | ||
| fi | ||
| } | ||
|
|
||
| if [ $1 -le 1 ]; then | ||
| # perform the below actions for rpm remove($1=0) or upgrade($1=1) operations | ||
| # remove pyc file generated by python | ||
| rm -rf @CPACK_PACKAGING_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBEXECDIR@/rocm_smi/__pycache__ | ||
| rm_logFolder | ||
| return_logrotateToOrigConfig | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.