Module src.pipelines.field_translator

Expand source code
# Copyright (c) 2019 Universidade da Coruña
# Authors:
#     - Jonatan Enes [main](jonatan.enes@udc.es, jonatan.enes.alvarez@gmail.com)
#     - Roberto R. Expósito
#     - Juan Touriño
#
# This file is part of the BDWatchdog framework, from
# now on referred to as BDWatchdog.
#
# BDWatchdog is free software: you can redistribute it
# and/or modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation, either version 3
# of the License, or (at your option) any later version.
#
# BDWatchdog is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with BDWatchdog. If not, see <http://www.gnu.org/licenses/>.


from __future__ import print_function
import sys


def eprint(*args, **kwargs):
    print(*args, file=sys.stderr, **kwargs)


def process_line(line):
    fields = line.split(",")

    # PROCESS INFO #
    if fields[0] == "PRC":
        time_interval_seconds = int(fields[3])
        fields[7] = str(format(1.0 * int(fields[7]) / time_interval_seconds, '.2f'))  # User
        fields[8] = str(format(1.0 * int(fields[8]) / time_interval_seconds, '.2f'))  # Kernel
        fields[9] = str(format(1.0 * int(fields[9]) / time_interval_seconds, '.2f'))  # Sleep
    if fields[0] == "PRM":
        fields[6] = str(format(int(fields[6]) / 1024.0, '.2f'))  # Virtual Memory
        fields[7] = str(format(int(fields[7]) / 1024.0, '.2f'))  # Resident Memory
        # fields[8] = str(format(int(fields[8]) / 1024.0, '.2f')) #Swap Memory
    if fields[0] == "PRD":
        time_interval = int(fields[3])
        fields[7] = str(format(int(fields[7]) * 512.0 / (time_interval * 1048576.0), '.2f'))  # Bandwidth Read
        fields[8] = str(format(int(fields[8]) * 512.0 / (time_interval * 1048576.0), '.2f'))  # Bandwidth Write
    if fields[0] == "PRN":
        time_interval = int(fields[3])
        fields[8] = str(
            format((int(fields[8]) * 8) / (time_interval * 1048576.0), '.2f'))  # Bandwidth out TCP in Mbps
        fields[10] = str(
            format((int(fields[10]) * 8) / (time_interval * 1048576.0), '.2f'))  # Bandwidth in TCP in Mbps
        fields[12] = str(
            format((int(fields[12]) * 8) / (time_interval * 1048576.0), '.2f'))  # Bandwidth out UDP in Mbps
        fields[14] = str(
            format((int(fields[14]) * 8) / (time_interval * 1048576.0), '.2f'))  # Bandwidth in UDP in Mbps
    if fields[0] == "NETHOGS":
        time_interval = int(fields[3])
        fields[7] = str(format(float(fields[7]) * 8 / time_interval, '.2f'))  # Bandwidth out in Mbps
        fields[8] = str(format(float(fields[8]) * 8 / time_interval, '.2f'))  # Bandwidth in Mbps

    # SYSTEM INFO #
    if fields[0] == "CPU":
        time_interval_seconds = int(fields[3])
        cpu_tick_per_second = int(fields[4])
        num_cpus = int(fields[5])

        total_time_spent = int(fields[6]) + int(fields[7]) + int(fields[8]) + int(fields[10])
        # system + user + user niced + wait

        usage = str(
            format(100.0 * total_time_spent / (num_cpus * cpu_tick_per_second * time_interval_seconds), '.2f'))
        fields.append(usage)

    if fields[0] == "cpu":
        time_interval_seconds = int(fields[3])
        cpu_tick_per_second = int(fields[4])
        total_ticks = time_interval_seconds * cpu_tick_per_second
        fields[6] = str(format(int(fields[6]) * 100.0 / total_ticks, '.2f'))  # kernel

        tmp_f7 = int(fields[7]) * 100.0 / total_ticks
        tmp_f8 = int(fields[8]) * 100.0 / total_ticks

        fields[7] = str(format(tmp_f7, '.2f'))  # user
        fields[8] = str(format(tmp_f7 + tmp_f8, '.2f'))  # Add the two users

        fields[9] = str(format(int(fields[9]) * 100.0 / total_ticks, '.2f'))  # idle
        fields[10] = str(format(int(fields[10]) * 100.0 / total_ticks, '.2f'))  # wait
    if fields[0] == "DSK":
        time_interval = int(fields[3])
        time_spent_resolving_miliseconds = int(fields[5])
        fields[5] = str(
            format(100.0 * time_spent_resolving_miliseconds / (time_interval * 1000), '.2f'))  # Disk usage in %
        fields[7] = str(format(int(fields[7]) * 512.0 / (time_interval * 1048576.0), '.2f'))  # Bandwidth Read
        fields[9] = str(format(int(fields[9]) * 512.0 / (time_interval * 1048576.0), '.2f'))  # Bandwidth Write
    if fields[0] == "MEM":
        page_size = int(fields[3])
        fields[4] = str(format(page_size * int(fields[4]) / 1048576.0, '.2f'))  # total memory
        fields[5] = str(format(page_size * int(fields[5]) / 1048576.0, '.2f'))  # free memory
        usage = str(format(100.0 * (1 - float(fields[5]) / float(fields[4])), '.2f'))
        fields.append(usage)
    if fields[0] == "SWP":
        page_size = int(fields[3])
        fields[4] = str(format(page_size * int(fields[4]) / 1048576.0, '.2f'))  # total swap
        fields[5] = str(format(page_size * int(fields[5]) / 1048576.0, '.2f'))  # free swap
    if fields[0] == "NET":
        time_interval = int(fields[3])
        interface_speed = int(fields[9])  # / 8.0 # Convert from Mbps to MBps
        interface_mode = int(fields[10])  # 0 -> half, 1 -> full
        fields[6] = str(format((int(fields[6]) * 8) / (time_interval * 1048576.0), '.2f'))  # Bandwidth in in Mbps
        fields[8] = str(format((int(fields[8]) * 8) / (time_interval * 1048576.0), '.2f'))  # Bandwidth out in Mbps

        if interface_mode == 0 and float(interface_speed) > 0:
            # Half duplex
            total_bandwidth = float(fields[6]) + float(fields[8])
            fields[9] = str(format(100.0 * total_bandwidth / interface_speed, '.2f'))
        elif interface_mode == 1 and float(interface_speed) > 0:
            # Full duplex
            total_bandwidth = float(fields[6]) + float(fields[8])
            fields[9] = str(format(100.0 * total_bandwidth / (2 * interface_speed), '.2f'))
        else:
            # Other, transport, network or loopback "devices"
            pass

    if fields[0] == "INFINIBAND":
        time_interval = int(fields[3])
        fields[6] = str(format((int(fields[6]) * 4) / (time_interval * 1048576.0), '.2f'))  # Bandwidth in in Mbps
        fields[7] = str(format((int(fields[7]) * 4) / (time_interval * 1048576.0), '.2f'))  # Bandwidth out in Mbps

    return str(",".join(fields)).replace("\n", "")


def behave_like_pipeline():
    try:
        # for line in fileinput.input():
        while True:
            line = sys.stdin.readline().rstrip('\n')
            # If code is changed back to a for loop, remove the rstrip part
            print(process_line(line))
    except (KeyboardInterrupt, IOError):
        # Exit silently
        pass
    except Exception as e:
        eprint("[FIELD TRANSLATOR] error : " + str(e))


def main():
    behave_like_pipeline()


if __name__ == "__main__":
    main()

Functions

def behave_like_pipeline()
Expand source code
def behave_like_pipeline():
    try:
        # for line in fileinput.input():
        while True:
            line = sys.stdin.readline().rstrip('\n')
            # If code is changed back to a for loop, remove the rstrip part
            print(process_line(line))
    except (KeyboardInterrupt, IOError):
        # Exit silently
        pass
    except Exception as e:
        eprint("[FIELD TRANSLATOR] error : " + str(e))
def eprint(*args, **kwargs)
Expand source code
def eprint(*args, **kwargs):
    print(*args, file=sys.stderr, **kwargs)
def main()
Expand source code
def main():
    behave_like_pipeline()
def process_line(line)
Expand source code
def process_line(line):
    fields = line.split(",")

    # PROCESS INFO #
    if fields[0] == "PRC":
        time_interval_seconds = int(fields[3])
        fields[7] = str(format(1.0 * int(fields[7]) / time_interval_seconds, '.2f'))  # User
        fields[8] = str(format(1.0 * int(fields[8]) / time_interval_seconds, '.2f'))  # Kernel
        fields[9] = str(format(1.0 * int(fields[9]) / time_interval_seconds, '.2f'))  # Sleep
    if fields[0] == "PRM":
        fields[6] = str(format(int(fields[6]) / 1024.0, '.2f'))  # Virtual Memory
        fields[7] = str(format(int(fields[7]) / 1024.0, '.2f'))  # Resident Memory
        # fields[8] = str(format(int(fields[8]) / 1024.0, '.2f')) #Swap Memory
    if fields[0] == "PRD":
        time_interval = int(fields[3])
        fields[7] = str(format(int(fields[7]) * 512.0 / (time_interval * 1048576.0), '.2f'))  # Bandwidth Read
        fields[8] = str(format(int(fields[8]) * 512.0 / (time_interval * 1048576.0), '.2f'))  # Bandwidth Write
    if fields[0] == "PRN":
        time_interval = int(fields[3])
        fields[8] = str(
            format((int(fields[8]) * 8) / (time_interval * 1048576.0), '.2f'))  # Bandwidth out TCP in Mbps
        fields[10] = str(
            format((int(fields[10]) * 8) / (time_interval * 1048576.0), '.2f'))  # Bandwidth in TCP in Mbps
        fields[12] = str(
            format((int(fields[12]) * 8) / (time_interval * 1048576.0), '.2f'))  # Bandwidth out UDP in Mbps
        fields[14] = str(
            format((int(fields[14]) * 8) / (time_interval * 1048576.0), '.2f'))  # Bandwidth in UDP in Mbps
    if fields[0] == "NETHOGS":
        time_interval = int(fields[3])
        fields[7] = str(format(float(fields[7]) * 8 / time_interval, '.2f'))  # Bandwidth out in Mbps
        fields[8] = str(format(float(fields[8]) * 8 / time_interval, '.2f'))  # Bandwidth in Mbps

    # SYSTEM INFO #
    if fields[0] == "CPU":
        time_interval_seconds = int(fields[3])
        cpu_tick_per_second = int(fields[4])
        num_cpus = int(fields[5])

        total_time_spent = int(fields[6]) + int(fields[7]) + int(fields[8]) + int(fields[10])
        # system + user + user niced + wait

        usage = str(
            format(100.0 * total_time_spent / (num_cpus * cpu_tick_per_second * time_interval_seconds), '.2f'))
        fields.append(usage)

    if fields[0] == "cpu":
        time_interval_seconds = int(fields[3])
        cpu_tick_per_second = int(fields[4])
        total_ticks = time_interval_seconds * cpu_tick_per_second
        fields[6] = str(format(int(fields[6]) * 100.0 / total_ticks, '.2f'))  # kernel

        tmp_f7 = int(fields[7]) * 100.0 / total_ticks
        tmp_f8 = int(fields[8]) * 100.0 / total_ticks

        fields[7] = str(format(tmp_f7, '.2f'))  # user
        fields[8] = str(format(tmp_f7 + tmp_f8, '.2f'))  # Add the two users

        fields[9] = str(format(int(fields[9]) * 100.0 / total_ticks, '.2f'))  # idle
        fields[10] = str(format(int(fields[10]) * 100.0 / total_ticks, '.2f'))  # wait
    if fields[0] == "DSK":
        time_interval = int(fields[3])
        time_spent_resolving_miliseconds = int(fields[5])
        fields[5] = str(
            format(100.0 * time_spent_resolving_miliseconds / (time_interval * 1000), '.2f'))  # Disk usage in %
        fields[7] = str(format(int(fields[7]) * 512.0 / (time_interval * 1048576.0), '.2f'))  # Bandwidth Read
        fields[9] = str(format(int(fields[9]) * 512.0 / (time_interval * 1048576.0), '.2f'))  # Bandwidth Write
    if fields[0] == "MEM":
        page_size = int(fields[3])
        fields[4] = str(format(page_size * int(fields[4]) / 1048576.0, '.2f'))  # total memory
        fields[5] = str(format(page_size * int(fields[5]) / 1048576.0, '.2f'))  # free memory
        usage = str(format(100.0 * (1 - float(fields[5]) / float(fields[4])), '.2f'))
        fields.append(usage)
    if fields[0] == "SWP":
        page_size = int(fields[3])
        fields[4] = str(format(page_size * int(fields[4]) / 1048576.0, '.2f'))  # total swap
        fields[5] = str(format(page_size * int(fields[5]) / 1048576.0, '.2f'))  # free swap
    if fields[0] == "NET":
        time_interval = int(fields[3])
        interface_speed = int(fields[9])  # / 8.0 # Convert from Mbps to MBps
        interface_mode = int(fields[10])  # 0 -> half, 1 -> full
        fields[6] = str(format((int(fields[6]) * 8) / (time_interval * 1048576.0), '.2f'))  # Bandwidth in in Mbps
        fields[8] = str(format((int(fields[8]) * 8) / (time_interval * 1048576.0), '.2f'))  # Bandwidth out in Mbps

        if interface_mode == 0 and float(interface_speed) > 0:
            # Half duplex
            total_bandwidth = float(fields[6]) + float(fields[8])
            fields[9] = str(format(100.0 * total_bandwidth / interface_speed, '.2f'))
        elif interface_mode == 1 and float(interface_speed) > 0:
            # Full duplex
            total_bandwidth = float(fields[6]) + float(fields[8])
            fields[9] = str(format(100.0 * total_bandwidth / (2 * interface_speed), '.2f'))
        else:
            # Other, transport, network or loopback "devices"
            pass

    if fields[0] == "INFINIBAND":
        time_interval = int(fields[3])
        fields[6] = str(format((int(fields[6]) * 4) / (time_interval * 1048576.0), '.2f'))  # Bandwidth in in Mbps
        fields[7] = str(format((int(fields[7]) * 4) / (time_interval * 1048576.0), '.2f'))  # Bandwidth out in Mbps

    return str(",".join(fields)).replace("\n", "")