www.blackholesurfer.com

CMDB professionals helping clients to skate the event horizon of a black hole (where your data disappears). Keeping you ahead of the game.
http://blackholesurfer.com/phpBB3/

Mongo Scan extended

http://blackholesurfer.com/phpBB3/viewtopic.php?f=23&t=76

Page 1 of 1

Mongo Scan extended

Posted: Thu Oct 15, 2020 10:55 am
by hudatolah

Code: Select all

// Copyright (c) 2013-2019 BMC Software Inc. All rights reserved.
tpl 1.14 module tengen.MongoDB_Extended;
metadata
    origin    := "TKU";
    tkn_name  := "MongoDB - Database Detail Pattern";
    tree_path := 'Structured Data Management Software', 'Nonrelational Database Management Systems', 'MongoDB', 'Extended discovery';
end metadata;
from System import Future 1.0;
from DiscoveryFunctions import DiscoveryFunctions 1.7;

pattern Server_DBDetails 4.1
    '''
        Pattern for discovering and modelling the MongoDB databases

        This pattern triggers off one of the three MongoDB Server Software Instances.
        It then proceeds to list the databases by excuting a command or listing
        the contents of the directory pointed to by db_path and then
        models each as a DatabaseDetail node.

        Supported Platforms:
        Windows
        UNIX

        Change History:
        2014-02-18 - Publisher name changed from 10gen to MongoDB (QM001816727)
        2014-06-18 - Removed duplicated 'MongoDB' wording from DatabaseDetail's type attribute. Key is unchanged (TKU-2289)
        2016-05-18 - Added WiredTiger storage extension to find databases in filesystem.
        2016-07-05 - Added command based discovery of databases (DRDC1-6285)
        2016-07-05 - Removed searching for .wt database files in favour of the
                     db.adminCommand('listDatabases') command (DRDC1-6285)
        2017-06-05 - Update for new Database kind
        2017-11-01 - Now utilises WiredTiger storage extension to find databases from specific .wt files (DRDC1-10134)
        2017-11-14 - Database nodes now attached to Replication Cluster node if host part of a cluster (DRDC1-9909)
        2019-02-12 - Updated pattern to use Future.runCommand and
                     DiscoveryFunctions.escapePath functions
        2019-03-13 - Updated UNIX command to list database files (DRDC1-12419)
    '''

    metadata
        categories       := "Nonrelational Database Management Systems";
        products         := "MongoDB";
        publishers       := "MongoDB";
        prior_publishers := "10gen";
        urls             := "http://www.mongodb.com/";
    end metadata;

    overview
        tags TKU, TKU_2019_03_01, tengen, MongoDB, Server;
    end overview;

    constants
        database_detail_type := "MongoDB Database";
    end constants;

    triggers
        on si := SoftwareInstance created, confirmed where
                     db_path matches regex "^(/|\w:)"
                     and
                     type in ["MongoDB Server", "MongoDB Config Server",
                              "MongoDB Shard Server"];
    end triggers;

    body
        host := model.host(si);

        if host.os_class = "Windows" then
            sep := '\\';
        else
            sep := '/';
        end if;

        database_names   := [];
        database_sizes   := table();
        detail_node_list := [];
        port             := "";
        serialnumber := none;
        sn := none;
               ran_cmdSN := "";
               cmdSN := "";

        if si.listening_ports then
            port := number.toText(si.listening_ports[0]);
        end if;

        // Obtain list of databases via command execution
        if si.install_root then
            append := '''--eval "printjson(db.adminCommand('listDatabases'))"''';
            appendSN := '''--eval "printjson(db.sumit.find(\{name:\"serialnumber\"}))"''';
            if port then
                append := "-port %port% %append%";
                appendSN := "-port %port% %appendSN%";
                log.info("appendSN'%appendSN%' looks like this");
             end if;
            if host.os_class = "Windows" then
                cmd := '''"%si.install_root%\\bin\\mongo.exe" admin %append%''';
                // *command_info_start
                // command_windows := '"<install_root>\bin\mongo.exe" admin -port <port> --eval "printjson(db.adminCommand('listDatabases'))"'
                // command_windows := '"<install_root>\bin\mongo.exe" admin --eval "printjson(db.adminCommand('listDatabases'))"'
                // reason := "Obtain list of databases"
                // when := "Only if installation root known"
                // *command_info_end
                ran_cmd := Future.runCommand(host, cmd, '%si.install_root%\\bin\\mongo.exe');

                ran_cmdSN := Future.runCommand(host, cmd, '%si.install_root%\\bin\\mongo.exe');
                log.info("Windows Host");
            else
                cmd := "%si.install_root%/bin/mongo admin %append%";
                cmdSN := "%si.install_root%/bin/mongo admin %appendSN%";
                // *command_info_start
                // command_unix    := '<install_root>/bin/mongo admin -port <port> --eval "printjson(db.adminCommand('listDatabases'))"'
                // command_unix    := '<install_root>/bin/mongo admin --eval "printjson(db.adminCommand('listDatabases'))"'
                // reason := "Obtain list of databases"
                // when := "Only if installation root known"
                // *command_info_end
                ran_cmd := Future.runCommand(host, cmd, '%si.install_root%/bin/mongo');
                ran_cmdSN := Future.runCommand(host, cmdSN, '%si.install_root%/bin/mongo');
                log.info("Line:113 ran_cmd = '%ran_cmd%' and ran_cmdSN = '%ran_cmdSN%'");
            end if;

            if (not ran_cmd or not ran_cmd.result
                or not ran_cmd.result has subword "name"
                or ran_cmd.result has subword "unauthorized")
                and host.os_class = "UNIX" then
                // Results might be:
                // MongoDB shell version: 2.4.13
                // connecting to: 127.0.0.1:27017/admin
                // { "ok" : 0, "errmsg" : "unauthorized" }
                // *command_info_start
                // command_unix    := 'PRIV_RUNCMD <install_root>/bin/mongo admin -port <port> --eval "printjson(db.adminCommand('listDatabases'))"'
                // command_unix    := 'PRIV_RUNCMD <install_root>/bin/mongo admin --eval "printjson(db.adminCommand('listDatabases'))"'
                // reason := "Obtain list of databases"
                // privileges := true
                // when := "Only if previous non-privileged command execution failed"
                // *command_info_end
                ran_cmd := Future.runCommand(host, 'PRIV_RUNCMD %cmd%', '%si.install_root%/bin/mongo');
                ran_cmdSN := Future.runCommand(host, 'PRIV_RUNCMD %cmdSN%', '%si.install_root%/bin/mongo');
                log.info("Line:133 %ran_cmdSN%");
            end if;

            if ran_cmd and ran_cmd.result then
                databases := text.split(ran_cmd.result, '}');
                for database in databases do
                    database_name := regex.extract(database,
                                                   regex '"name"\s*:\s*"(\S+)"',
                                                   raw '\1');
                    if database_name then
                        list.append(database_names, database_name);
                        size_bytes := regex.extract(database,
                                                    regex '"sizeOnDisk"\s*:\s*(\d+)',
                                                    raw '\1');
                        if size_bytes then
                            database_sizes[database_name] := text.toNumber(size_bytes);
                        end if;
                    end if;
                end for;
            end if;
        end if;

// dhudsky -> 
            sn_value := none;

            if ran_cmdSN and ran_cmdSN.result then
                serialnumber := text.split(ran_cmdSN.result, '}');
                    sn_value := regex.extract(ran_cmdSN.result,
                                                   regex '"name"\s*:\s*"(\S+)"',
                                                   raw '\1');
                           log.info("Got SerialNumber: %sn_value%");
            end if;
log.info("Line:154 Got this result '%ran_cmdSN%'");
// dhudsky <-- 

        // Obtain list of databases via directory listing
        if not database_names then
            esc_db_path := DiscoveryFunctions.escapePath(host, si.db_path);
            // Search for .ns files in the database directory. It is not possible to perform a
            // similar search for .wt files due to such files having strange names like
            // 'collection-45-3947809740892176921.wt' or 'index-15-3947809740892176921.wt'
            if host.os_class = "Windows" then
                if si.__directory_per_db then
                    cmd := 'dir /s %esc_db_path%\\*.ns';
                else
                    cmd := 'dir %esc_db_path%\\*.ns';
                end if;
                // *command_info_start
                // command_windows := 'dir /s "<db_path>\*.ns"'
                // command_windows := 'dir "<db_path>\*.ns"'
                // when := "Only if unable to obtain databases from previous command";
                // reason := "Obtain database names from directory listing"
                // *command_info_end
                dir_cmd := discovery.runCommand(host, cmd);
                if dir_cmd and dir_cmd.result then
                    // The following regex would return a list such as ['eol','hrd']
                    database_names := regex.extractAll(dir_cmd.result,
                                                       regex "\d\s([\w\s-]+)\.ns");
                end if;
            else
                if si.__directory_per_db then
                    cmd := 'ls -R %esc_db_path% | grep "\.ns"';
                else
                    cmd := 'ls %esc_db_path% | grep "\.ns"';
                end if;
                // *command_info_start
                // command_unix := 'ls -R "<db_path>" | grep ".ns"'
                // command_unix := 'ls "<db_path>" | grep ".ns"'
                // when := "Only if unable to obtain databases from previous command";
                // reason := "Obtain database names from directory listing"
                // *command_info_end
                dir_cmd := discovery.runCommand(host, cmd);
                if dir_cmd and dir_cmd.result then
                    // The following regex would return a list such as ['eol','hrd']
                    database_names := regex.extractAll(dir_cmd.result,
                                                       regex "([\w-]+)\.ns");
                end if;
            end if;
        end if;

        // If unable to extract database names from the names of .ns files
        // attempt to extract them from the content of the _mdb_catalog.wt file
        if not database_names then
            // *filepath_info_start
            // filepath_unix := "<db_path>/_mdb_catalog.wt"
            // filepath_windows := "<db_path>\\_mdb_catalog.wt"
            // reason := "Extract database names from content"
            // when := "Only if database names not already obtained"
            // *filepath_info_end
            wt_file := discovery.fileGet(host, '%si.db_path%%sep%_mdb_catalog.wt');
            if wt_file and wt_file.content then
                database_names := regex.extractAll(wt_file.content, regex 'ns\x00\x18\x00\x00\x00([\w\-_]+)\.\w+');
            end if;
        end if;

        // If unable to extract database names from the content of _mdb_catalog.wt file
        // attempt to extract them from the content of the WiredTiger.wt file
        if not database_names then
            // *filepath_info_start
            // filepath_unix := "<db_path>/WiredTiger.wt"
            // filepath_windows := "<db_path>\\WiredTiger.wt"
            // reason := "Extract database names from content"
            // when := "Only if database names not already obtained"
            // *filepath_info_end
            wt_file := discovery.fileGet(host, '%si.db_path%%sep%WiredTiger.wt');
            if wt_file and wt_file.content then
                database_names := regex.extractAll(wt_file.content, regex '"ns"\s+\:\s+"([\w\-_]+)\.\w+"');
            end if;
        end if;

        // Remove duplicated database names
        if database_names then
            unique_database_names := [];
            for database_name in database_names do
                if database_name not in unique_database_names then
                    list.append(unique_database_names, database_name);
                end if;
            end for;
            database_names := unique_database_names;
        end if;

        // Although 'MongoDB' as vendor has already been omitted from the DatabaseDetail's type attribute,
        // it is being retained in the key to avoid unnecessarily altering the same.
        key_database_detail_type := "MongoDB %database_detail_type%";

        // Ascertain if the host is part of a Master/Slave or Replica Set Replication Cluster
        msrc_list := search(in si traverse ContainedSoftware:SoftwareContainment:SoftwareContainer:SoftwareCluster
                            where type = 'MongoDB Server Replication Cluster');
        if size(msrc_list) = 1 then
            hosting_node := msrc_list[0];
        else
            hosting_node := si;
        end if;

        for database_name in database_names do
            database_detail_key := "%database_name%/%key_database_detail_type%/%hosting_node.key%";
            db_detail := model.Database(name                        := database_name,
                                              instance              := database_name,
                                              logical_database_type := database_detail_type,
                                              key                   := database_detail_key,
                                              type                  := database_detail_type);
            log.info("Database modeled for database %database_name%");
            if database_name in database_sizes then
                db_detail.size_bytes := database_sizes[database_name];
            end if;
            list.append(detail_node_list, db_detail);
            model.setRemovalGroup(db_detail, "MongoDB_db_detail_nodes");
        end for;
/// dhudsky -- > 
           qresult := none;

     for i in ran_cmdSN do
           log.info("Line: before if ran_cmdSN '%i%'"); 
     end for; 

if not ran_cmdSN = "None" then
            qresult := ran_cmdSN.serialnumber;
            log.info("RESULTS: '%qresult%'");
            db_detail.serial := '%qresult%';
            model.addDisplayAttribute (db_detail,"serial");
        end if;

log.info("cmd = '%cmd%'");
log.info("database_name = '%database_name%'");
log.info("database_names = '%database_names%'");
log.info("append = '%append%'");
log.info("appendSN = '%appendSN%'");
log.info("cmd = '%cmd%'");
log.info("cmdSN = '%cmdSN%'");
log.info("database_names = '%database_names%'");
log.info("databases = '%databases%'");
log.info("db_detail.size_bytes = '%db_detail.size_bytes%'");
log.info("dir_cmd = '%dir_cmd%'");
log.info("ran_cmd = '%ran_cmd%'");
log.info("ran_cmdSN = '%ran_cmdSN%'");
log.info("append = '%append%'");
log.info("appendSN = '%appendSN%'");
log.info("database_detail_key = '%database_detail_key%'");
log.info("database_names = '%database_names%'");
log.info("db_detail = '%db_detail%'");
log.info("db_detail.serial = '%db_detail.serial%'");
log.info("esc_db_path = '%esc_db_path%'");
log.info("hosting_node = '%hosting_node%'");
log.info("port = '%port%'");
log.info("qresult = '%qresult%'");
log.info("sep = '%sep%'");
log.info("unique_database_names = '%unique_database_names%'");
log.info("wt_file = '%wt_file%'");
log.info("qresult = '%qresult%'");
log.info("database_detail_type = '%database_detail_type%'");
log.info("database_names   = '%database_names%'");
log.info("database_sizes   = '%database_sizes%'");
log.info("detail_node_list = '%detail_node_list%'");
log.info("host = '%host%'");
log.info("key_database_detail_type = '%key_database_detail_type%'");
log.info("msrc_list = '%msrc_list%'");
log.info("on si = '%si%'");
log.info("port             = '%port%'");

/// < -- dhudsky

        // Creating relationship with the SI/Cluster to all DB Detail nodes together
        // This automatically deletes the relationships with details which are absent
        containment_rels := model.uniquerel.Detail(ElementWithDetail := hosting_node,
                                                   Detail := detail_node_list,
                                                   type := database_detail_type);

    end body;
end pattern;

Re: Mongo Scan extended

Posted: Fri Nov 06, 2020 3:19 pm
by hudatolah
I added a regex edit to the query:

Code: Select all

 sn_value := none;

            if ran_cmdSN and ran_cmdSN.result then
                serialnumber := text.split(ran_cmdSN.result, '}');
                    sn_value := regex.extract(ran_cmdSN.result,
                                                   regex '"name"\s*:\s*"(\S+)"',
                                                   raw '\1');
                           log.info("Got SerialNumber: %sn_value%");
            end if;
But eventually changed the CMDB Synch pattern to do this and scan for MNG prefix to process the serial number:

Code: Select all

tpl 1.5 module CMDB.Extension.Database_Augment_CTI;


metadata
    origin := 'Schwab';
    tree_path := 'Schwab','CMDB Sync','Database';
end metadata;

from CMDB.SoftwareInstance_SoftwareServer import SoftwareInstance_SoftwareServer 4.2;// edit  
from CMDB.Database import Database 2.3;

syncmapping Database_Augment 2.0
    """
    Update/Modify values for BMC_Database class
    This pattern has a dependency on:
    tpl 1.14 module AssetInfoFromSQL_Oracle
    tpl 1.14 module AssetInfoFromSQL_MSSQL 
    Please make sure to update versions and mappings should they change
    in relation to pattern DBSerialNumber 1.5
    """
    overview
        tags CMDB, Extension;
    end overview;

    mapping from Database.database_node as database_node
    end mapping;

    body
        database_ci := Database.database_ci;                           //Edit
        hosting_node := SoftwareInstance_SoftwareServer.hosting_node;  //Edit: Assign value to hosting_node

        //if database_node.type = "Oracle Database" or database_node.type = "SQL Server Database" then
		//Added teradata to if statement
		if database_node.type = "Oracle Database" or database_node.type = "SQL Server Database" or database_node.type = "Teradata Database" then		  
                   if database_node.serial then
			database_ci.SerialNumber := "%database_node.serial%";
                   end if;
	  end if;

sn := none;

         if database_node.type = "MongoDB Database" then
                        database_ci.ParentCITag := "%hosting_node.local_fqdn%";
			if database_node.name has substring '-MNG' then
				sn := regex.extract(database_node.name, regex'^(.*)-MNG(.*)$', raw'\2');
				database_ci.SerialNumber := "%sn%";
                        end if;
         end if;
    end body;
end syncmapping;
All times are UTC-05:00
Page 1 of 1
Powered by phpBB® Forum Software © phpBB Limited
https://www.phpbb.com/