Skip to content

Commit

Permalink
Merge branch 'develop' into develop
Browse files Browse the repository at this point in the history
  • Loading branch information
@tpurschke
tpurschke committed Jan 2, 2024
2 parents 0dc7234 + 597d7f7 commit fc8fa71
Show file tree
Hide file tree
Showing 40 changed files with 750 additions and 274 deletions.
1 change: 1 addition & 0 deletions inventory/group_vars/all.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,7 @@ fworch_log_dir: "/var/log/{{ product_name }}"
fworch_log_lock_dir: "/var/{{ product_name }}/lock"
fworch_mw_lockfile: "{{ fworch_log_lock_dir }}/FWO.Middleware.Server_log.lock"
fworch_ui_lockfile: "{{ fworch_log_lock_dir }}/FWO.Ui_log.lock"
fworch_api_importer_lockfile: "{{ fworch_log_lock_dir }}/importer_api_log.lock"

### apache: both package and dir name (needed both on ui and api hosts):
webserver_package_name: apache2
Expand Down
105 changes: 78 additions & 27 deletions roles/api/files/replace_metadata.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1008,7 +1008,11 @@
{
"role": "modeller",
"permission": {
"check": {},
"check": {
"app_id": {
"_in": "x-hasura-editable-owners"
}
},
"columns": [
"common_service",
"is_interface",
Expand Down Expand Up @@ -1077,7 +1081,11 @@
"reason",
"creation_date"
],
"filter": {},
"filter": {
"app_id": {
"_in": "x-hasura-editable-owners"
}
},
"check": {}
},
"comment": ""
Expand All @@ -1087,7 +1095,11 @@
{
"role": "modeller",
"permission": {
"filter": {}
"filter": {
"app_id": {
"_in": "x-hasura-editable-owners"
}
}
},
"comment": ""
}
Expand Down Expand Up @@ -1166,7 +1178,11 @@
{
"role": "modeller",
"permission": {
"check": {},
"check": {
"app_id": {
"_in": "x-hasura-editable-owners"
}
},
"columns": [
"id",
"is_deleted",
Expand Down Expand Up @@ -1264,7 +1280,11 @@
"group_type",
"creation_date"
],
"filter": {},
"filter": {
"app_id": {
"_in": "x-hasura-editable-owners"
}
},
"check": {}
},
"comment": ""
Expand All @@ -1281,7 +1301,11 @@
{
"role": "modeller",
"permission": {
"filter": {}
"filter": {
"app_id": {
"_in": "x-hasura-editable-owners"
}
}
},
"comment": ""
}
Expand Down Expand Up @@ -1718,7 +1742,11 @@
{
"role": "modeller",
"permission": {
"check": {},
"check": {
"app_id": {
"_in": "x-hasura-editable-owners"
}
},
"columns": [
"is_global",
"name",
Expand Down Expand Up @@ -1779,7 +1807,11 @@
"port_end",
"proto_id"
],
"filter": {},
"filter": {
"app_id": {
"_in": "x-hasura-editable-owners"
}
},
"check": null
},
"comment": ""
Expand All @@ -1789,7 +1821,11 @@
{
"role": "modeller",
"permission": {
"filter": {}
"filter": {
"app_id": {
"_in": "x-hasura-editable-owners"
}
}
},
"comment": ""
}
Expand Down Expand Up @@ -1904,7 +1940,11 @@
{
"role": "modeller",
"permission": {
"check": {},
"check": {
"app_id": {
"_in": "x-hasura-editable-owners"
}
},
"columns": [
"is_global",
"comment",
Expand Down Expand Up @@ -1965,7 +2005,11 @@
"id",
"creation_date"
],
"filter": {},
"filter": {
"app_id": {
"_in": "x-hasura-editable-owners"
}
},
"check": {}
},
"comment": ""
Expand All @@ -1975,7 +2019,11 @@
{
"role": "modeller",
"permission": {
"filter": {}
"filter": {
"app_id": {
"_in": "x-hasura-editable-owners"
}
}
},
"comment": ""
}
Expand Down Expand Up @@ -9050,6 +9098,18 @@
}
}
},
{
"name": "selected_connections",
"using": {
"foreign_key_constraint_on": {
"column": "app_id",
"table": {
"name": "selected_connections",
"schema": "modelling"
}
}
}
},
{
"name": "selected_objects",
"using": {
Expand Down Expand Up @@ -9236,24 +9296,15 @@
"last_recert_check"
],
"filter": {
"_and": [
"_or": [
{
"_or": [
{
"tenant_id": {
"_is_null": true
}
},
{
"tenant_id": {
"_eq": "x-hasura-tenant-id"
}
}
]
"tenant_id": {
"_is_null": true
}
},
{
"app_id_external": {
"_in": "x-hasura-visible-owners"
"tenant_id": {
"_eq": "x-hasura-tenant-id"
}
}
]
Expand Down
16 changes: 15 additions & 1 deletion roles/common/tasks/install_syslog.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,21 @@
{{ fworch_log_dir }}/database.log
{{ fworch_log_dir }}/ldap.log
{{ fworch_log_dir }}/api.log
{{ fworch_log_dir }}/importer-api.log
{{ fworch_log_dir }}/importer-api.log {
compress
maxage 7
rotate 99
maxsize 4096k
missingok
copytruncate
sharedscripts
prerotate
{{ fworch_home }}/scripts/acquire_lock.py {{ fworch_api_importer_lockfile }} >/dev/null 2>&1
endscript
postrotate
{{ fworch_home }}/scripts/release_lock.py {{ fworch_api_importer_lockfile }} >/dev/null 2>&1
endscript
}
{{ fworch_log_dir }}/audit.log
{{ fworch_log_dir }}/alert.log
{{ fworch_log_dir }}/webhook.log
Expand Down
122 changes: 101 additions & 21 deletions roles/importer/files/importer/fwo_log.py
View file
Original file line number Diff line number Diff line change
@@ -1,31 +1,107 @@
import logging
from sys import stdout
import sys
import fwo_globals
#from fwo_globals import global_debug_level
import logging
import asyncio
import time
from asyncio import Semaphore

class LogLock:
semaphore = Semaphore(value=1)

async def handle_log_lock():
# Initialize values
lock_file_path = "/var/fworch/lock/importer_api_log.lock"
log_owned_by_external = False
stopwatch = time.time()

while True:
try:
with open(lock_file_path, "a+") as file:
# Jump to the beginning of the file
file.seek(0)
# Read the file content
lock_file_content = file.read().strip()
# Forcefully release lock after timeout
if log_owned_by_external and time.time() - stopwatch > 10:
file.write("FORCEFULLY RELEASED\n")
stopwatch = -1
LogLock.semaphore.release()
log_owned_by_external = False
# GRANTED - lock was granted by us
elif lock_file_content.endswith("GRANTED"):
# Request lock if it is not already requested by us
# (in case of restart with log already granted)
if not log_owned_by_external:
await LogLock.semaphore.acquire()
stopwatch = time.time()
log_owned_by_external = True
# REQUESTED - lock was requested by log swap process
elif lock_file_content.endswith("REQUESTED"):
# only request lock if it is not already requested by us
if not log_owned_by_external:
await LogLock.semaphore.acquire()
stopwatch = time.time()
log_owned_by_external = True
file.write("GRANTED\n")
# RELEASED - lock was released by log swap process
elif lock_file_content.endswith("RELEASED"):
# only release lock if it was formerly requested by us
if log_owned_by_external:
stopwatch = -1
LogLock.semaphore.release()
log_owned_by_external = False
except Exception as e:
pass
# Wait a second
time.sleep(1)

# Used to accquire lock before log processing
class LogFilter(logging.Filter):
async def acquire_lock():
LogLock.semaphore.acquire()
def filter(self, record):
# Acquire lock
asyncio.run(LogFilter.acquire_lock())
# Return True to allow the log record to be processed
return True

# Used to release lock after log processing
class LogHandler(logging.StreamHandler):
async def release_lock():
LogLock.semaphore.release()
def emit(self, record):
# Call the parent class's emit method to perform the actual logging
super().emit(record)
# Release lock
asyncio.run(LogHandler.release_lock())

def getFwoLogger():
debug_level=int(fwo_globals.debug_level)
if debug_level>=1:
llevel = logging.DEBUG
debug_level = int(fwo_globals.debug_level)
if debug_level >= 1:
log_level = logging.DEBUG
else:
llevel = logging.INFO
log_level = logging.INFO

logger = logging.getLogger() # use root logger
logHandler = logging.StreamHandler(stream=stdout)
logformat = "%(asctime)s [%(levelname)-5.5s] [%(filename)-10.10s:%(funcName)-10.10s:%(lineno)4d] %(message)s"
logHandler.setLevel(llevel)
handlers = [logHandler]
logging.basicConfig(format=logformat, datefmt="%Y-%m-%dT%H:%M:%S%z", handlers=handlers, level=llevel)
logger.setLevel(llevel)
logger = logging.getLogger()
log_handler = LogHandler(stream=sys.stdout)
log_filter = LogFilter()

# set log level for noisy requests/connectionpool module to WARNING:
log_format = "%(asctime)s [%(levelname)-5.5s] [%(filename)-10.10s:%(funcName)-10.10s:%(lineno)4d] %(message)s"
log_handler.setLevel(log_level)
log_handler.addFilter(log_filter)
handlers = [log_handler]

logging.basicConfig(format=log_format, datefmt="%Y-%m-%dT%H:%M:%S%z", handlers=handlers, level=log_level)
logger.setLevel(log_level)

# Set log level for noisy requests/connectionpool module to WARNING:
connection_log = logging.getLogger("urllib3.connectionpool")
connection_log.setLevel(logging.WARNING)
connection_log.propagate = True

if debug_level>8:
logger.debug ("debug_level=" + str(debug_level) )

if debug_level > 8:
logger.debug("debug_level=" + str(debug_level))

return logger


Expand All @@ -37,10 +113,14 @@ def getFwoAlertLogger(debug_level=0):
llevel = logging.INFO

logger = logging.getLogger() # use root logger
logHandler = logging.StreamHandler(stream=stdout)
log_handler = LogHandler(stream=sys.stdout)
log_filter = LogFilter()

logformat = "%(asctime)s %(message)s"
logHandler.setLevel(llevel)
handlers = [logHandler]
log_handler.setLevel(llevel)
log_handler.addFilter(log_filter)
handlers = [log_handler]

logging.basicConfig(format=logformat, datefmt="", handlers=handlers, level=llevel)
logger.setLevel(llevel)

Expand Down
Loading

0 comments on commit fc8fa71

Please sign in to comment.