diff --git a/.gitignore b/.gitignore deleted file mode 100644 index e43b0f9..0000000 --- a/.gitignore +++ /dev/null @@ -1 +0,0 @@ -.DS_Store diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index d427698..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "devdocs-theme"] - path = devdocs-theme - url = https://github.com/citrix/devdocs-theme.git diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..bed3e2c --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,7 @@ +### CITRIX TOOL LICENSE AGREEMENT + +Use of this Citrix software is subject to the Citrix license covering the specific edition of the Citrix product with which you will be using this software. Citrix’s standard end-user license agreement (EULA) for its on-premises software and hardware offerings and its standard end-user service agreement (EUSA) for its Citrix Cloud and other SaaS offerings are available at https://www.citrix.com/buy/licensing/agreements.html. Your use of this software is limited to use in connection with the Citrix product(s) to which you are licensed. +Certain third-party software may be provided with this software that is subject to separate license conditions. The licenses are located in the third-party licenses file accompanying this component or in the corresponding license files available at www.citrix.com . + +Citrix and other marks are trademarks and/or registered trademarks of Citrix Systems, Inc. in the U.S. and other countries. + diff --git a/README.md b/README.md new file mode 100644 index 0000000..11e36c1 --- /dev/null +++ b/README.md @@ -0,0 +1,27 @@ +![Citrix Logo](media/Citrix_Logo_Trademark.png) + +# Citrix ADC scripts for migrating and converting Citrix ADC configuration with deprecated features + +## Description + +When you migrate from a Citrix ADC version with deprecated features, you may lose some of the configuration. Citrix provides you scripts to avoid such configuration loss when you are migrating from an old version with deprecated features to the newer version. + +This repository contains the following scripts: + +- [`tdToPartition.pl`](td-to-ap/tdToPartition.pl): The script for migrating the traffic domain configuration on a Citrix ADC to the admin partition configuration. For more information on how to use the script, see [Migrating traffic domain configuration on a Citrix ADC to admin partition configuration](td-to-ap/migration-script-td.md). + +- [`check_invalid_config`](nspepi/check_invalid_config): Pre-validation script to check if any deprecated functionality that is removed from Citrix ADC release version 13.1 is still used in the configuration. For more information on how to use the script, see [Scripts for pre-validating and converting deprecated features](nspepi/validation-conversion-script.md). + +- [`NSPEPI`](nspepi/nspepi): The script that converts deprecated commands or features to non-deprecated commands or features. For more information on how to use the script, see [Scripts for pre-validating and converting deprecated features](nspepi/validation-conversion-script.md). + + +## Questions + +For questions and support, the following channels are available: + +- [Citrix Discussion Forum](https://discussions.citrix.com/) + + +## Licensing + +The Citrix ADC scripts are licensed with [CITRIX TOOL LICENSE](LICENSE.md). \ No newline at end of file diff --git a/devdocs-theme b/devdocs-theme deleted file mode 160000 index aa7990f..0000000 --- a/devdocs-theme +++ /dev/null @@ -1 +0,0 @@ -Subproject commit aa7990fc4ab6ce924327f6a9c65eedf0226bf4ae diff --git a/docs/index.md b/docs/index.md deleted file mode 100644 index 3934c42..0000000 --- a/docs/index.md +++ /dev/null @@ -1,5 +0,0 @@ -# Citrix ADC Ansible Module Documentation - -This project implements a set of Ansible modules for configuring Citrix ADC appliances. Users of these modules can create, edit, update, and delete configuration objects on a Citrix ADC appliance. - -The code is licensed under **GPL**. The authoritative repository, which includes the code and the detailed documentation, is on GitHub at [Citrix ADC Ansible Module](https://github.com/citrix/citrix-adc-ansible-modules). diff --git a/media/Citrix_Logo_Trademark.png b/media/Citrix_Logo_Trademark.png new file mode 100644 index 0000000..dad3bcc Binary files /dev/null and b/media/Citrix_Logo_Trademark.png differ diff --git a/mkdocs.yml b/mkdocs.yml deleted file mode 100644 index c0196b1..0000000 --- a/mkdocs.yml +++ /dev/null @@ -1,72 +0,0 @@ -# Project information -site_name: Citrix NetScaler Ansible Module Documentation -site_title: Developer Docs -site_description: Documentation for Citrix NetScaler Ansible Modules 0.1 -site_url: https://developer-docs.citrix.com - -# Feedback - -issues_uri: https://github.com/citrix/devdocs-issue-collector/issues/new -assignees_uri: - - ajitchelat -labels_uri: - - documentation - -# Copyright -copyright: '© 1999-2020 Citrix Systems, Inc. All rights reserved.' - -extra_css: - - 'assets/stylesheets/extra.css' - - https://use.fontawesome.com/releases/v5.12.1/css/all.css - - -# Configuration -theme: - name: null - custom_dir: devdocs-theme - - # 404 page - static_templates: - - 404.html - - # Don't include MkDocs' JavaScript - include_search_page: false - search_index_only: true - - # Default values, taken from mkdocs_theme.yml - language: en - features: - - tabs: false - palette: - primary: teal - accent: teal - font: false - favicon: https://docs.citrix.com/assets/images/favicon.ico - logo: https://developer-docs.citrix.com/_static/logo.svg - icon: fontawesome - -extra: - social: - - icon: fontawesome/brands/youtube - link: https://www.youtube.com/channel/UCiOupk9QF6jdk3EDKTHDykA - - icon: fontawesome/brands/github-alt - link: https://github.com/citrix - - icon: fontawesome/brands/twitter - link: https://twitter.com/citrixdeveloper - - icon: fontawesome/brands/linkedin - link: https://www.linkedin.com/company/citrix/ - -markdown_extensions: - - admonition - - codehilite - - toc: - permalink: true - -plugins: - - search: - separator: '[\s\-\.]+' - -# Page tree -nav: - - Home: index.md - - Questions?: https://discussions.citrix.com/forum/1385-citrix-developer-exchange/ diff --git a/nspepi/check_invalid_config b/nspepi/check_invalid_config new file mode 100755 index 0000000..57b2034 --- /dev/null +++ b/nspepi/check_invalid_config @@ -0,0 +1,40 @@ +#!/usr/bin/perl + +# Copyright 2021 Citrix Systems, Inc. All rights reserved. +# Use of this software is governed by the license terms, if any, +# which accompany or are included with this software. + +$ENV{PATH} = '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin'.$ENV{PATH}; + +use File::Basename; + +my $number_args = $#ARGV + 1; +if ($number_args != 1) { + print "Usage: check_invalid_config \n"; + exit; +} + +my $config_file = $ARGV[0]; +if (not -e $config_file) { + print "No such file: $config_file\n"; + exit; +} + +my($filename, $dir_path) = fileparse($config_file); + +my $exit_status = system("python2 /netscaler/nspepi2/config_check_main.py -f $config_file"); +if ($exit_status != 0) { + print "Error in checking config file: $exit_status"; + exit; +} +my $invalid_config_file = $dir_path."/issues_".$filename; + +# Checks whether any command is present in the file +if (!(-z $invalid_config_file)) { + print "\nThe following configuration lines will get errors in 13.1 and both they and dependent configuration will be removed from the configuration:\n"; + system("cat $invalid_config_file"); + print "\nThe nspepi upgrade tool can be useful in converting your configuration - see the documentation at https://docs.citrix.com/en-us/citrix-adc/current-release/appexpert/policies-and-expressions/introduction-to-policies-and-exp/converting-policy-expressions-nspepi-tool.html. Use the latest tool version for the most complete and up-to-date version.\n"; +} else { + print "\nNo issue detected with the configuration.\n"; +} +### End check_invalid_config script diff --git a/nspepi/nspepi b/nspepi/nspepi new file mode 100755 index 0000000..c867d5d --- /dev/null +++ b/nspepi/nspepi @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# Copyright 2021 Citrix Systems, Inc. All rights reserved. +# Use of this software is governed by the license terms, if any, +# which accompany or are included with this software. + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +exec $DIR/nspepi2/nspepi_main.py "$@" diff --git a/nspepi/nspepi2/check_classic_configs.py b/nspepi/nspepi2/check_classic_configs.py new file mode 100644 index 0000000..ca3bb79 --- /dev/null +++ b/nspepi/nspepi2/check_classic_configs.py @@ -0,0 +1,624 @@ +#!/usr/bin/env python2 + +# Copyright 2021 Citrix Systems, Inc. All rights reserved. +# Use of this software is governed by the license terms, if any, +# which accompany or are included with this software. + +import copy + +import cli_lex +import nspepi_common as common +import check_classic_expr +from nspepi_parse_tree import * + + +def check_configs_init(): + """Initialize global variables uses by this module""" + global policy_entities_names + global classic_entities_names + global named_expr + named_expr = {} + policy_entities_names = set() + classic_entities_names = set() + # Register built-in named expressions. + NamedExpression.register_built_in_named_exprs() + + +def remove_quotes(val): + """ + Helper function to remove the surrounding + quotes from a CLI parameter. + val - CLI parameter that needs quotes removed. + Returns the dequoted CLI parameter + """ + result = val + if val.startswith('"') or val.startswith("'"): + lexer = cli_lex.Lexer() + lexer.input(val) + token = lexer.token() + assert token.type == "NON_KEY" + result = token.value + return result + + +def is_classic_named_expr_present(expr): + """ + Helper function to check that + classic expression names present in the + given expression or not. + expr - Expression in which classic + expression names need to be found. + Returns True if classic named expression is present, + otherwise returns False. + """ + lexer = cli_lex.Lexer() + lexer.input(expr) + classic_expr_info_list = [] + while True: + next_token = lexer.adv_expr_token() + if not next_token: + break + token_value = str(next_token) + if token_value in NamedExpression.built_in_named_expr: + return True + elif token_value in classic_entities_names: + return True + return False + +class CheckConfig(object): + """Base class to check the config""" + + @staticmethod + def check_pos_expr(commandParseTree, pos): + """ + Check the expression present at a given position + commandParseTree - the parse tree to modify + pos - the position of the parameter to modify + If the expression is classic, then invalid + flag would be set. + """ + rule_node = commandParseTree.positional_value(pos) + rule_expr = rule_node.value + converted_expr = check_classic_expr.check_classic_expr(rule_expr) + if converted_expr is None: + logging.error('Error in checking command : ' + + str(commandParseTree)) + else: + # converted_expr will have quotes and rule_expr will not have + # quotes. Since we are comparing these 2 expressions, removing + # quotes from converted_expr. + converted_expr = remove_quotes(converted_expr) + if converted_expr != rule_expr: + # expression is converted, this is classic. + commandParseTree.set_invalid() + if is_classic_named_expr_present(converted_expr): + commandParseTree.set_invalid() + return commandParseTree + + @staticmethod + def check_keyword_expr(commandParseTree, keywordName): + """ + Check the expression present as a value of + the given keyword name. + commandParseTree - the parse tree to modify + keywordName - the name of the keyword parameter to modify + If the expression is classic, then invalid + flag would be set. + """ + if not commandParseTree.keyword_exists(keywordName): + return commandParseTree + rule_node = commandParseTree.keyword_value(keywordName) + rule_expr = rule_node[0].value + converted_expr = check_classic_expr.check_classic_expr(rule_expr) + if converted_expr is None: + logging.error('Error in checking command : ' + + str(commandParseTree)) + else: + # converted_expr will have quotes and rule_expr will not have + # quotes. Since we are comparing these 2 expressions, removing + # quotes from converted_expr. + converted_expr = remove_quotes(converted_expr) + if converted_expr != rule_expr: + # expression is converted, this is classic. + commandParseTree.set_invalid() + if is_classic_named_expr_present(converted_expr): + commandParseTree.set_invalid() + return commandParseTree + + @staticmethod + def register_policy_entity_name(commandParseTree): + """ Add the entity name in the global list.""" + name = commandParseTree.positional_value(0).value.lower() + policy_entities_names.add(name) + + @staticmethod + def register_classic_entity_name(commandParseTree): + """ Add the classic entity name in the classic global list.""" + name = commandParseTree.positional_value(0).value.lower() + classic_entities_names.add(name) + + +@common.register_class_methods +class CacheRedirection(CheckConfig): + """ Handle CR feature """ + + # Classic built-in policy names + built_in_policies = [ + "bypass-non-get", + "bypass-cache-control", + "bypass-dynamic-url", + "bypass-urltokens", + "bypass-cookie" + ] + + + @common.register_for_cmd("add", "cr", "policy") + def check_policy(self, commandParseTree): + """ + Checks classic CR policy. + """ + policy_name = commandParseTree.positional_value(0).value + pol_obj = common.Policy(policy_name, self.__class__.__name__) + common.pols_binds.store_policy(pol_obj) + """ + If action field is not set, then it is classic policy, + else it is an advanced policy. + """ + if commandParseTree.keyword_exists('action'): + return [] + else: + return [commandParseTree] + + @common.register_for_cmd("bind", "cr", "vserver") + def check_cr_vserver_bind(self, bind_parse_tree): + """ + Handles CR vserver bind command. + bind cr vserver -policyName + -priority -gotoPriorityExpression + """ + if not bind_parse_tree.keyword_exists('policyName'): + return [] + + policy_name = bind_parse_tree.keyword_value("policyName")[0].value + class_name = self.__class__.__name__ + policy_type = common.pols_binds.get_policy(policy_name).module + # When policy is CR policy. + if policy_type == class_name: + # check for classic built-in policy. + if policy_name in self.built_in_policies: + return [bind_parse_tree] + + return [] + + +@common.register_class_methods +class SSL(CheckConfig): + """ Handle SSL feature """ + + @common.register_for_cmd("add", "ssl", "policy") + def check_policy(self, commandParseTree): + """ + Check classic SSL policy. + """ + + commandParseTree = SSL.check_keyword_expr(commandParseTree, 'rule') + if commandParseTree.invalid: + return [commandParseTree] + return [] + + +@common.register_class_methods +class APPFw(CheckConfig): + """ Handle APPFw feature """ + + @common.register_for_cmd("add", "appfw", "policy") + def check_policy(self, commandParseTree): + """ + Check classic AppFw policy + """ + commandParseTree = APPFw.check_pos_expr(commandParseTree, 1) + if commandParseTree.invalid: + return [commandParseTree] + return [] + + +@common.register_class_methods +class Patset(CheckConfig): + """ Patset entity """ + + @common.register_for_cmd("add", "policy", "patset") + def register_name(self, commandParseTree): + Patset.register_policy_entity_name(commandParseTree) + if commandParseTree.keyword_exists('indexType'): + return [commandParseTree] + return [] + + +@common.register_class_methods +class Dataset(CheckConfig): + """ Dataset entity """ + + @common.register_for_cmd("add", "policy", "dataset") + def register_name(self, commandParseTree): + Dataset.register_policy_entity_name(commandParseTree) + if commandParseTree.keyword_exists('indexType'): + return [commandParseTree] + return [] + + +@common.register_class_methods +class HTTP_CALLOUT(CheckConfig): + """ HTTP callout entity """ + + @common.register_for_cmd("add", "policy", "httpCallout") + def register_name(self, commandParseTree): + HTTP_CALLOUT.register_policy_entity_name(commandParseTree) + return [] + + +@common.register_class_methods +class StringMap(CheckConfig): + """ String map entity """ + + @common.register_for_cmd("add", "policy", "stringmap") + def register_name(self, commandParseTree): + StringMap.register_policy_entity_name(commandParseTree) + return [] + + +@common.register_class_methods +class NSVariable(CheckConfig): + """ NS Variable entity """ + + @common.register_for_cmd("add", "ns", "variable") + def register_name(self, commandParseTree): + NSVariable.register_policy_entity_name(commandParseTree) + return [] + + +@common.register_class_methods +class EncryptionKey(CheckConfig): + """ Encryption key entity """ + + @common.register_for_cmd("add", "ns", "encryptionKey") + def register_name(self, commandParseTree): + EncryptionKey.register_policy_entity_name(commandParseTree) + return [] + + +@common.register_class_methods +class HMACKey(CheckConfig): + """ HMAC key entity """ + + @common.register_for_cmd("add", "ns", "hmacKey") + def register_name(self, commandParseTree): + HMACKey.register_policy_entity_name(commandParseTree) + return [] + + +@common.register_class_methods +class NamedExpression(CheckConfig): + """ Handle Named expression feature """ + + # Built-in classic named expression names + built_in_named_expr = { + "ns_true", + "ns_false", + "ns_non_get", + "ns_cachecontrol_nostore", + "ns_cachecontrol_nocache", + "ns_header_pragma", + "ns_header_cookie", + "ns_ext_cgi", + "ns_ext_asp", + "ns_ext_exe", + "ns_ext_cfm", + "ns_ext_ex", + "ns_ext_shtml", + "ns_ext_htx", + "ns_url_path_cgibin", + "ns_url_path_exec", + "ns_url_path_bin", + "ns_url_tokens", + "ns_ext_not_gif", + "ns_ext_not_jpeg", + "ns_cmpclient", + "ns_slowclient", + "ns_content_type", + "ns_msword", + "ns_msexcel", + "ns_msppt", + "ns_css", + "ns_xmldata", + "ns_mozilla_47", + "ns_msie" + } + + @staticmethod + def register_built_in_named_exprs(): + """ + Register built-in classic Named expression names in + classic_entities_names. + """ + for classic_exp_name in NamedExpression.built_in_named_expr: + classic_entities_names.add(classic_exp_name) + + @common.register_for_cmd("add", "policy", "expression") + def check_policy_expr(self, commandParseTree): + """ + Classic named expression name is not + valid for advanced expression if: + 1. It the name is same as one of the Policy + entity (patset/dataset/stringmap/ + variable/hmacKey/EncriptionKey/callout) name. + 2. it doesn't start with ASCII alphabetic character or underscore. + 3. it has characters other than ASCII alphanumerics + or underscore characters. + 4. it is equal to a advanced policy expression reserved word (prefix identifier or + enum value) + """ + reserved_word_list = set( + [ # Advanced policy expression prefix list + "subscriber", + "connection", + "analytics", + "diameter", + "target", + "server", + "radius", + "oracle", + "extend", + "client", + "mysql", + "mssql", + "false", + "true", + "text", + "smpp", + "icap", + "http", + "url", + "sys", + "sip", + "ica", + "dns", + "aaa", + "re", + "xp", + "ce" + ]) + + expr_name = commandParseTree.positional_value(0).value + expr_rule = commandParseTree.positional_value(1).value + named_expr[expr_name] = expr_rule + lower_expr_name = expr_name.lower() + if (((lower_expr_name in reserved_word_list) or + (re.match('^[a-z_][a-z0-9_]*$', lower_expr_name) is None) or + (lower_expr_name in policy_entities_names))): + logging.error(("Expression name {} is invalid for advanced " + "expression: names must begin with an ASCII " + "alphabetic character or underscore and must " + "contain only ASCII alphanumerics or underscores" + " and shouldn't be name of another policy entity" + "; words reserved for policy use may not be used;" + " underscores will be substituted for any invalid" + " characters in corresponding advanced name") + .format(expr_name)) + + if commandParseTree.keyword_exists('clientSecurityMessage'): + NamedExpression.register_classic_entity_name(commandParseTree) + return [] + + original_tree = copy.deepcopy(commandParseTree) + commandParseTree = NamedExpression \ + .check_pos_expr(commandParseTree, 1) + + if commandParseTree.invalid: + """ + Add the commands in the global list which will be used to + check whether any other expression is using these named + expressions. + """ + NamedExpression.register_policy_entity_name(commandParseTree) + NamedExpression.register_classic_entity_name(original_tree) + else: + NamedExpression.register_policy_entity_name(original_tree) + return [] + + +@common.register_class_methods +class HTTPProfile(CheckConfig): + """ Handle HTTP Profile """ + + @common.register_for_cmd("add", "ns", "httpProfile") + @common.register_for_cmd("set", "ns", "httpProfile") + def check_spdy(self, commandParseTree): + """ + Check if spdy parameter present in HTTP profile. + Syntax: + """ + if commandParseTree.keyword_exists('spdy'): + return [commandParseTree] + return [] + + +@common.register_class_methods +class ContentSwitching(CheckConfig): + """ Check Content Switching feature """ + + @common.register_for_cmd("add", "cs", "policy") + def check_cs_policy(self, commandParseTree): + if commandParseTree.keyword_exists('action'): + return [] + if commandParseTree.keyword_exists('rule'): + if commandParseTree.keyword_exists('domain'): + return [commandParseTree] + else: + original_cmd = copy.deepcopy(commandParseTree) + commandParseTree = ContentSwitching \ + .check_keyword_expr(commandParseTree, 'rule') + if commandParseTree.invalid: + return [original_cmd] + elif commandParseTree.keyword_exists('url'): + return [commandParseTree] + elif commandParseTree.keyword_exists('domain'): + return [commandParseTree] + + return [] + + +@common.register_class_methods +class CMP(CheckConfig): + """ + Checks CMP feature commands. + """ + + # Classic built-in policy names. + built_in_policies = [ + "ns_cmp_content_type", + "ns_cmp_msapp", + "ns_cmp_mscss", + "ns_nocmp_mozilla_47", + "ns_nocmp_xml_ie" + ] + + @common.register_for_cmd("set", "cmp", "parameter") + def set_cmp_parameter(self, cmp_param_tree): + if cmp_param_tree.keyword_exists("policyType"): + self._initial_cmp_parameter = \ + cmp_param_tree.keyword_value("policyType")[0].value.lower() + if self._initial_cmp_parameter == "classic": + return [cmp_param_tree] + return [] + + @common.register_for_cmd("set", "cmp", "policy") + def set_cmp_policy(self, cmp_policy_tree): + policy_name = cmp_policy_tree.positional_value(0).value + if policy_name in self.built_in_policies: + return [cmp_policy_tree] + return [] + + @common.register_for_cmd("add", "cmp", "policy") + def check_cmp_policy(self, cmp_policy_tree): + original_cmd = copy.deepcopy(cmp_policy_tree) + CheckConfig.check_keyword_expr(cmp_policy_tree, 'rule') + if cmp_policy_tree.invalid: + return [original_cmd] + return [] + + @common.register_for_cmd("bind", "cmp", "global") + def check_cmp_global_bind(self, bind_cmd_tree): + """ + Checks CMP policy bindings to cmp global. + """ + # If state keyword is present then it is a + # classic binding. + if bind_cmd_tree.keyword_exists("state"): + return [bind_cmd_tree] + + policy_name = bind_cmd_tree.positional_value(0).value + if policy_name in self.built_in_policies: + return [bind_cmd_tree] + return [] + + +@common.register_class_methods +class CLITransformFilter(CheckConfig): + """ + Checks Filter feature + """ + + @common.register_for_cmd("add", "filter", "action") + def check_filter_action(self, action_parse_tree): + """ + Check Filter action + """ + return [action_parse_tree] + + @common.register_for_cmd("add", "filter", "policy") + def check_filter_policy(self, policy_parse_tree): + """ + Check Filter policy + """ + return [policy_parse_tree] + + @common.register_for_cmd("bind", "filter", "global") + def check_filter_global_bindings(self, bind_parse_tree): + """ + Check Filter global binding + """ + return [bind_parse_tree] + + @common.register_for_cmd("add", "filter", "htmlinjectionvariable") + @common.register_for_cmd("set", "filter", "htmlinjectionvariable") + @common.register_for_cmd("set", "filter", "htmlinjectionparameter") + @common.register_for_cmd("set", "filter", "prebodyInjection") + @common.register_for_cmd("set", "filter", "postbodyInjection") + def check_filter_htmlinjection_command(self, cmd_parse_tree): + """ + Check Filter HTMLInjection command + """ + return [cmd_parse_tree] + + +@common.register_class_methods +class Rewrite(CheckConfig): + """ + Check rewrite action + """ + + @common.register_for_cmd("add", "rewrite", "action") + def check_rewrite_action(self, tree): + if tree.keyword_exists('pattern'): + return [tree] + if tree.keyword_exists('bypassSafetyCheck'): + return [tree] + return [] + + +@common.register_class_methods +class LB(CheckConfig): + """ + Check LB persistence rule + """ + + @common.register_for_cmd("add", "lb", "vserver") + def check_rewrite_action(self, commandParseTree): + commandParseTree = LB.check_keyword_expr(commandParseTree, 'rule') + if commandParseTree.invalid: + return [commandParseTree] + return [] + + +@common.register_class_methods +class SureConnect(CheckConfig): + """ + Check SureConnect commands + """ + + @common.register_for_cmd("add", "sc", "policy") + @common.register_for_cmd("set", "sc", "parameter") + def check_sc_policy(self, tree): + return [tree] + + +@common.register_class_methods +class PriorityQueuing(CheckConfig): + """ + Check PriorityQueuing commands + """ + + @common.register_for_cmd("add", "pq", "policy") + def check_sc_policy(self, tree): + return [tree] + + +@common.register_class_methods +class HDoSP(CheckConfig): + """ + Check HTTP Denial of Service Protection commands + """ + + @common.register_for_cmd("add", "dos", "policy") + def check_sc_policy(self, tree): + return [tree] diff --git a/nspepi/nspepi2/check_classic_expr.py b/nspepi/nspepi2/check_classic_expr.py new file mode 100644 index 0000000..c6902b4 --- /dev/null +++ b/nspepi/nspepi2/check_classic_expr.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python2 + +# Copyright 2021 Citrix Systems, Inc. All rights reserved. +# Use of this software is governed by the license terms, if any, +# which accompany or are included with this software. + +import logging +import subprocess +from nspepi_parse_tree import CLIParseTreeNode +import nspepi_common as common + + +def check_classic_expr(classic_expr): + tree_obj = CLIParseTreeNode() + info_msg = 'INFO: Expression is not converted' + \ + ' - most likely it is a valid advanced expression' + warn_msg = 'WARNING: Line numbers which has ' + \ + 'more than 8191 characters length: 0' + try: + nspepi_tool_path = common.get_nspepi_tool_path() + """Error message will be in the staring of + output, whereas warning and info messages + will be present in the last.""" + nspepi_tool_output = subprocess.check_output( + ['perl', nspepi_tool_path, '-e', classic_expr], + shell=False, stderr=subprocess.STDOUT) + """ old nspepi tool adds newline character at the end + of the converted string, so remove that character.""" + nspepi_tool_output = nspepi_tool_output.rstrip() + except subprocess.CalledProcessError as exc: + # Log the command which is failing + logging.error(exc) + # Log the error message + logging.error(exc.output) + return None + if nspepi_tool_output.startswith('ERROR:'): + """old nspepi tool throws "ERROR: Expression is in blocked list + of conversion" error for vpn client security expression. + We are not removing client security expressions, so these + are valid expressions.""" + nspepi_tool_output = tree_obj.normalize(classic_expr, True) + elif nspepi_tool_output.endswith(info_msg): + """old nspepi tool didn't convert the expression, + so return input expression""" + nspepi_tool_output = tree_obj.normalize(classic_expr, True) + elif nspepi_tool_output.endswith(warn_msg): + logging.warning(nspepi_tool_output) + """ If expression has more than 8191 characters, old nspepi + tool gives warning message at the end of the output. + + old nspepi tool output: + WARNING: Total number of warnings due to + expressions length greater than 8191 characters: 1 + WARNING: Line numbers which has more than 8191 characters length: 0 + + Removing warning message from the output + """ + expr_end_pos = nspepi_tool_output.find("WARNING") + nspepi_tool_output = nspepi_tool_output[0:expr_end_pos] + nspepi_tool_output = nspepi_tool_output.rstrip() + + return nspepi_tool_output diff --git a/nspepi/nspepi2/cli_lex.py b/nspepi/nspepi2/cli_lex.py new file mode 100644 index 0000000..73d79f6 --- /dev/null +++ b/nspepi/nspepi2/cli_lex.py @@ -0,0 +1,369 @@ +#!/usr/bin/env python2 + +# Copyright 2021 Citrix Systems, Inc. All rights reserved. +# Use of this software is governed by the license terms, if any, +# which accompany or are included with this software. + +""" + CLI lexical analyzer. +""" + +import logging + + +class LexToken(object): + """ + Class to represent the token. + Instance variables: + type - Token type + value - Token value + lineno - line number where the + token value is in the data parsed + lexpos - Points to the token end position + """ + + def __init__(self, token_type, token_value, lineno, lex_pos): + self.type = token_type + self.value = token_value + self.lineno = lineno + self.lexpos = lex_pos + + def __str__(self): + return self.value + + def __repr__(self): + return "LexToken({},{},{},{})".format( + self.type, + self.value, + self.lineno, + self.lexpos + ) + + +class Lexer(object): + """ + CLI lexical analyzer. + Instance variables: + data - data to be parsed + lex_pos - Points to the current position during parsing + length - length of the remaining data that + has to be parsed + token_value - value of the current token during tokenization + """ + + def __init__(self): + self._data = None + self._lex_pos = 0 + self._length = 0 + self._token_value = "" + + def input(self, command): + """ + Sets the lexical analyzer with the data that has to be parsed. + command - command that has to be parsed + """ + self._data = command + self._lex_pos = 0 + self._length = len(self._data) + self._token_value = "" + + def token(self): + """ + Returns next token in a CLI command as a LexToken object. + If there is no more data to be parsed, returns None. + """ + start_pos = self._lex_pos + + # Handling spaces that comes around the tokens + while self._length > 0 and (self._data[self._lex_pos] in " \t\n"): + self.advance_token() + + # Ignoring comments + if self._length > 0 and self._data[self._lex_pos] == '#': + self.advance_token(self._length) + + # When there is no more input to be parsed, returns None + if self._length <= 0: + return None + + # Identifies Token type + token_type = None + if self._data[self._lex_pos] == '-': + token_type = "KEY_ARG" + # Removing '-' from the starting of the keyword + self.advance_token() + + # quotes are not allowed in keywords + self._token_value = "" + while self._length > 0: + if self._data[self._lex_pos] in " \t\n": + return LexToken(token_type, self._token_value, 1, + self._lex_pos - 1) + else: + self.advance_and_append_token(self._data[self._lex_pos]) + return LexToken(token_type, self._token_value, 1, + self._lex_pos - 1) + else: + token_type = "NON_KEY" + + start_pos = self._lex_pos + + # Handling q quote + qquote_start_delims = "/{<|~$^+=&%@`?" + qquote_end_delims = {"{": "}", "<": ">"} + + if (self._length > 2 and self._data[start_pos] == 'q' and + self._data[start_pos + 1] in qquote_start_delims): + qquote_end_char = qquote_end_delims.get(self._data[start_pos + 1], + self._data[start_pos + 1]) + qquote_end_index = self._data.find(qquote_end_char, start_pos + 2) + if (qquote_end_index != -1 and + (qquote_end_index == len(self._data) - 1 or + self._data[qquote_end_index + 1] in " \t\n")): + next_token = LexToken("NON_KEY", + self._data[start_pos + 2: + qquote_end_index], + 1, qquote_end_index) + self.advance_token(len(next_token.value) + 3) + return next_token + + state = " " + parenthesis_counter = 0 + self._token_value = "" + while self._length > 0: + if self._data[self._lex_pos] in "\"'": + if self._data[self._lex_pos] == state: + # end of quotes + state = " " + """ + If token starts with quotes and the corresponding ending + quotes appear, then this will be the end of the token. + If token doesn't start with the quote, + then this will be the end of the quote but not the + end of the token. + """ + if self._data[self._lex_pos] == self._data[start_pos]: + # Removing end quotes by not appending the character + self.advance_token() + break + self.advance_and_append_token(self._data[self._lex_pos]) + elif state in "\"'": + # single quote within double quote or vice versa + self.advance_and_append_token(self._data[self._lex_pos]) + else: + # now inside quotes + state = self._data[self._lex_pos] + # Removing starting quotes + if self._lex_pos != start_pos: + self.advance_and_append_token( + self._data[self._lex_pos]) + else: + self.advance_token() + elif self._data[self._lex_pos] in " \t\n": + if state == " " and parenthesis_counter == 0: + break + # This case occurs when whitespace appears inside quotes + self.advance_and_append_token(self._data[self._lex_pos]) + elif self._data[self._lex_pos] == "(": + self.advance_and_append_token(self._data[self._lex_pos]) + if state not in "\"'": + parenthesis_counter += 1 + elif self._data[self._lex_pos] == ")": + self.advance_and_append_token(self._data[self._lex_pos]) + if state not in "\"'": + if parenthesis_counter > 0: + parenthesis_counter -= 1 + else: + self.advance_token(self._length) + token_type = "ERROR" + logging.error("Data: {}".format(self._data)) + logging.error("Unbalanced closed parenthesis") + break + elif self._data[self._lex_pos] == "\\": + # backslashes are escapes inside quotes + if state in "\"'": + if self._length == 1: + # \\ followed by end of the command + self.advance_and_append_token( + self._data[self._lex_pos]) + token_type = "ERROR" + logging.error("Data: {}".format(self._data)) + logging.error("Blackslashes inside quotes are " + "followed by end of the command") + break + if self._data[self._lex_pos + 1] == 't': + self.advance_and_append_token('\t', 2) + elif self._data[self._lex_pos + 1] == 'n': + self.advance_and_append_token('\n', 2) + elif self._data[self._lex_pos + 1] == 'r': + self.advance_and_append_token('\r', 2) + elif self._data[self._lex_pos + 1] in "'\"\\": + self.advance_and_append_token( + self._data[self._lex_pos + 1], 2) + else: + self.advance_and_append_token( + self._data[self._lex_pos]) + else: + self.advance_and_append_token(self._data[self._lex_pos]) + else: + self.advance_and_append_token(self._data[self._lex_pos]) + + if state in "\"'" or parenthesis_counter > 0: + # error token for not matching with any rule + token_type = "ERROR" + logging.error("Data: {}".format(self._data)) + logging.error("Unbalanced parenthesis or quotes") + next_token = LexToken(token_type, self._token_value, 1, + self._lex_pos - 1) + return next_token + + def advance_token(self, number=1): + """ + This function increments the class instance variable lex_pos and + decrements the length variable by the number that is passed as + argument. + number - number by which the increment has to be done + """ + self._lex_pos += number + self._length -= number + + def advance_and_append_token(self, token_char, number=1): + """ + This function advances current position by the number that is + passed as argument and appends the token value based on token_char. + number - number by which current position is incremented, + by default it is 1 + token_char - character that has to be appened to the token value + """ + self.advance_token(number) + self._token_value += token_char + + @staticmethod + def adv_ident_char(ch): + """ Helper function to check whether + character is an Advanced identifier character: + letter, underscore, or digit. + ch - character to check + """ + return (ch == "_") or ch.isdigit() or ch.isalpha() + + def adv_expr_token(self): + """ + This function is used to tokenize an Advanced + expression. + Note that currently this only recognizes a subset of the token types. + Returns next token as LexToken object. + If there is no more data to be parsed, returns None. + """ + # Handling spaces that comes around the tokens + while self._length > 0 and (self._data[self._lex_pos] in " \t\r\n"): + self.advance_token() + + # When there is no more input to be parsed, returns None + if self._length <= 0: + return None + + # Identifies Token type + token_type = "OTHER" + + start_pos = self._lex_pos + + state = " " + self._token_value = "" + while self._length > 0: + if state == "REGEX" and self._data[self._lex_pos] == regex_end: + # End of regex + state = " " + self.advance_token() + break + elif self._data[self._lex_pos] in "\"'": + if self._data[self._lex_pos] == state: + # end of quotes + state = " " + # Removing end quotes by not appending the character + self.advance_token() + break + elif state in "\"'": + # single quote within double quote or vice versa + self.advance_and_append_token(self._data[self._lex_pos]) + elif state == "REGEX": + self.advance_and_append_token(self._data[self._lex_pos]) + else: + # now inside quotes + state = self._data[self._lex_pos] + # Removing starting quotes + self.advance_token() + token_type = "STRING" + elif state == " " and self._data[self._lex_pos] in " \t\r\n": + # Whitespace ends most tokens + break + elif (state == " " and self._length >= 5 and + self._data[self._lex_pos:self._lex_pos+2].lower() == 're' + and not Lexer.adv_ident_char(self._data[self._lex_pos+2])): + # Start of regex + state = "REGEX" + token_type = "REGEX" + regex_end = self._data[self._lex_pos+2] + self.advance_token(3) + elif ((self._data[self._lex_pos] == "_") or + (self._data[self._lex_pos].isalpha())): + if ((state not in "\"'" and state != "IDENTIFIER" and + state != "REGEX" and self._lex_pos != start_pos)): + # End of "other" + break + elif state not in "\"'" and state != "REGEX": + # start of an identifier + state = "IDENTIFIER" + token_type = "IDENTIFIER" + self.advance_and_append_token(self._data[self._lex_pos]) + else: + self.advance_and_append_token(self._data[self._lex_pos]) + elif (not Lexer.adv_ident_char(self._data[self._lex_pos])): + if state == "IDENTIFIER": + state = " " + break + else: + # More of identifier + self.advance_and_append_token(self._data[self._lex_pos]) + elif self._data[self._lex_pos] == "\\": + # backslashes are escapes inside quotes + if state in "\"'": + if self._length == 1: + # \\ followed by end of the expression + self.advance_and_append_token( + self._data[self._lex_pos]) + token_type = "ERROR" + logging.error("Data: {}".format(self._data)) + logging.error("Blackslashes inside quotes are " + "followed by end of the expression") + break + if self._data[self._lex_pos + 1] == 't': + self.advance_and_append_token('\t', 2) + elif self._data[self._lex_pos + 1] == 'n': + self.advance_and_append_token('\n', 2) + elif self._data[self._lex_pos + 1] == 'r': + self.advance_and_append_token('\r', 2) + elif self._data[self._lex_pos + 1] in "'\"\\": + self.advance_and_append_token( + self._data[self._lex_pos + 1], 2) + else: + self.advance_and_append_token( + self._data[self._lex_pos]) + else: + self.advance_and_append_token(self._data[self._lex_pos]) + else: + self.advance_and_append_token(self._data[self._lex_pos]) + + if state in "\"'": + # error token for not matching with any rule + token_type = "ERROR" + logging.error("Data: {}".format(self._data)) + logging.error("Unbalanced quotes") + elif state == "REGEX": + # error token for not matching with any rule + token_type = "ERROR" + logging.error("Data: {}".format(self._data)) + logging.error("Unterminated regex") + next_token = LexToken(token_type, self._token_value, 1, + self._lex_pos - 1) + return next_token diff --git a/nspepi/nspepi2/cli_yacc.py b/nspepi/nspepi2/cli_yacc.py new file mode 100644 index 0000000..a336fce --- /dev/null +++ b/nspepi/nspepi2/cli_yacc.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python2 + +# Copyright 2021 Citrix Systems, Inc. All rights reserved. +# Use of this software is governed by the license terms, if any, +# which accompany or are included with this software. + +import ply.yacc as yacc +import cli_lex +from nspepi_parse_tree import * +import logging + +tokens = ('NON_KEY', 'KEY_ARG') + + +def p_command_empty(p): + 'command : empty' + p[0] = None + + +def p_command(p): + 'command : command_name positional_parameters keyword_parameters' + p[0] = p[1] + p[0].add_positional_list(p[2]) + p[0].add_keyword_list(p[3]) + + +def p_command_name(p): + 'command_name : op group ot' + p[0] = CLICommand(p[1], p[2], p[3]) + + +def p_command_name_no_ot(p): + 'command_name : op group' + p[0] = CLICommand(p[1], p[2], "") + + +def p_op(p): + 'op : NON_KEY' + logging.debug("CLI lex op: " + p[1]) + p[0] = p[1] + + +def p_group(p): + 'group : NON_KEY' + logging.debug("CLI lex group: " + p[1]) + p[0] = p[1] + + +def p_ot(p): + 'ot : NON_KEY' + logging.debug("CLI lex ot: " + p[1]) + p[0] = p[1] + + +def p_empty(p): + 'empty :' + pass + + +def p_pos_params(p): + 'positional_parameters : positional_parameters NON_KEY' + logging.debug("CLI lex pos: " + p[2]) + p[0] = p[1] + [CLIPositionalParameter(p[2])] + + +def p_pos_empty_param(p): + 'positional_parameters : empty' + p[0] = [] + + +def p_keyword_params(p): + 'keyword_parameters : keyword_parameters keyword_parameter' + p[0] = p[1] + [p[2]] + + +def p_keyword_empty_param(p): + 'keyword_parameters : empty' + p[0] = [] + + +def p_key_param(p): + 'keyword_parameter : keyword keyword_value' + p[0] = CLIKeywordParameter(p[1]) + p[0].add_value_list(p[2]) + + +def p_keyword(p): + 'keyword : KEY_ARG' + logging.debug("CLI lex key: " + p[1]) + p[0] = CLIKeywordName(p[1]) + + +def p_key_val(p): + 'keyword_value : keyword_value NON_KEY' + logging.debug("CLI lex key val: " + p[2]) + p[0] = p[1] + [p[2]] + + +def p_key_empty_val(p): + 'keyword_value : empty' + p[0] = [] + + +# This is for syntax errors +def p_error(p): + if p is None: + p = "EOL" + logging.error("CLI syntax error at " + str(p)) + + +_lexer = None +_parser = None + + +def cli_yacc_init(): + """ Initialize CLI command parser + """ + global _lexer + global _parser + _lexer = cli_lex.Lexer() + _parser = yacc.yacc(debug=False, write_tables=False) + + +def cli_yacc_parse(cmd, lineno): + """ Parse a CLI command. + cmd - the CLI command + lineno - the line number of the command + returns the parse tree or None if either "empty" line or syntax error + """ + tree = _parser.parse(cmd, lexer=_lexer) + if tree is not None: + tree.original_line = cmd + tree.lineno = lineno + return tree diff --git a/nspepi/nspepi2/config_check_main.py b/nspepi/nspepi2/config_check_main.py new file mode 100755 index 0000000..97e9278 --- /dev/null +++ b/nspepi/nspepi2/config_check_main.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python2 + +# Copyright 2021 Citrix Systems, Inc. All rights reserved. +# Use of this software is governed by the license terms, if any, +# which accompany or are included with this software. + +""" +Checks whether config file contains invalid +epressions and features. + +Dependency packages: PLY, pytest +""" + +# Ensure that the version string conforms to PEP 440: +# https://www.python.org/dev/peps/pep-0440/ +__version__ = "1.0" + +import re +import argparse +import glob +import importlib +import logging +import logging.handlers +import os +import os.path +import sys +from inspect import cleandoc +import inspect + +import cli_yacc +import nspepi_common as common + +import check_classic_configs + +# Log handlers that need to be saved from call to call +file_log_handler = None +console_log_handler = None + + +def setup_logging(log_file_name, file_log_level): + """ + Sets up logging for the program. + + Args: + log_file_name: The name of the log file + file_log_level: The level of logs to put in the file log + """ + global file_log_handler + global console_log_handler + # create logger + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + # if called multiple times, remove existing handlers + logger.removeHandler(file_log_handler) + logger.removeHandler(console_log_handler) + # create file handler and roll logs if needed + exists = os.path.isfile(log_file_name) + file_log_handler = logging.handlers.RotatingFileHandler(log_file_name, + mode='a', + backupCount=9) + if exists: + file_log_handler.doRollover() + # set the file log handler level + file_log_handler.setLevel(file_log_level) + # create console handler that sees even info messages + console_log_handler = logging.StreamHandler() + console_log_handler.setLevel(logging.INFO) + # create formatters and add them to the handlers + fh_format = logging.Formatter('%(asctime)s: %(levelname)s - %(message)s') + ch_format = logging.Formatter('%(levelname)s - %(message)s') + file_log_handler.setFormatter(fh_format) + console_log_handler.setFormatter(ch_format) + # add the handlers to the logger + logger.addHandler(file_log_handler) + logger.addHandler(console_log_handler) + + +def output_line(line, outfile, verbose): + """ + Output a (potentially) converted line. + + Args: + line: the line to output + outfile: Output file to write converted commands + verbose: True iff converted commands should also be output to console + """ + outfile.write(line) + if verbose: + logging.info(line.rstrip()) + + +def check_for_removed_expression(cmd, outfile, verbose): + """ + Checks for the expressions which are removed. + Args: + cmd: NS config command to be checked + outfile: Output file to write commands using removed config + verbose: True iff converted commands should also be output to console + """ + if re.search(r'\bSYS\s*\.\s*EVAL_CLASSIC_EXPR\s*\(', + cmd, re.IGNORECASE): + output_line(cmd, outfile, verbose) + return + + body_expr = re.compile(r'\bHTTP\s*\.\s*REQ\s*\.\s*BODY\b\s*', re.IGNORECASE) + cmd_len = len(cmd) + for match in re.finditer(body_expr, cmd): + start_index = match.start() + length = match.end() - match.start() + if (((start_index + length) >= cmd_len) or + (cmd[start_index + length] != '(')): + output_line(cmd, outfile, verbose) + return + + if re.search(r'\b((Q\.HOSTNAME)|(Q\.TRACKING)|' + '(Q\.METHOD)|(Q\.URL)|(Q\.VERSION)|' + '(Q\.CONTENT_LENGTH)|(Q\.HEADER)|' + '(Q\.IS_VALID)|(Q\.DATE)|' + '(Q\.COOKIE)|(Q\.BODY)|(Q\.TXID)|' + '(Q\.CACHE_CONTROL)|(Q\.USER)|' + '(Q\.IS_NTLM_OR_NEGOTIATE)|' + '(Q\.FULL_HEADER)|' + '(Q\.LB_VSERVER)|(Q\.CS_VSERVER))', + cmd, re.IGNORECASE): + output_line(cmd, outfile, verbose) + return + + if re.search(r'\b((S\.VERSION)|(S\.STATUS)|' + '(S\.STATUS_MSG)|(S\.IS_REDIRECT)|' + '(S\.IS_INFORMATIONAL)|(S\.IS_SUCCESSFUL)|' + '(S\.IS_CLIENT_ERROR)|(S\.IS_SERVER_ERROR)|' + '(S\.TRACKING)|(S\.HEADER)|(S\.FULL_HEADER)|' + '(S\.IS_VALID)|(S\.DATE)|(S\.BODY)|' + '(S\.SET_COOKIE)|(S\.SET_COOKIE2)|' + '(S\.CONTENT_LENGTH)|' + '(S\.CACHE_CONTROL)|(S\.TXID)|(S\.MEDIA))', + cmd, re.IGNORECASE): + output_line(cmd, outfile, verbose) + return + + +def check_config_file(infile, outfile, verbose): + """ + Process ns config file passed in argument and report the classic and + removed commands. + + Args: + infile: NS config file to be converted + outfile: Output file to write commands using removed config + verbose: True iff converted commands should also be output to console + """ + cli_yacc.cli_yacc_init() + # Register handler methods for various commands + currentfile = os.path.abspath(inspect.getfile(inspect.currentframe())) + currentdir = os.path.dirname(currentfile) + for module in glob.glob(os.path.join(currentdir, 'check_classic_configs.py')): + importlib.import_module(os.path.splitext(os.path.basename(module))[0]) + # call methods registered to be called before the start of processing + # config file. + for m in common.init_methods: + m.method(m.obj) + lineno = 0 + for cmd in infile: + lineno += 1 + parsed_tree = cli_yacc.cli_yacc_parse(cmd, lineno) + if parsed_tree is not None: + # construct dictionary key to look up registered method to call to + # parse and transform the command to be emitted + # Registered method can return either string or tree. + key = " ".join(parsed_tree.get_command_type()).lower() + if key in common.dispatchtable: + for m in common.dispatchtable[key]: + # Since, we are only checking the config and not + # converting to advanced, so list will contains + # at most only one command. + output = m.method(m.obj, parsed_tree) + if len(output) == 0: + check_for_removed_expression(cmd, outfile, verbose) + else: + output_line(cmd, outfile, verbose) + else: + check_for_removed_expression(cmd, outfile, verbose) + else: + check_for_removed_expression(cmd, outfile, verbose) + + +def main(): + desc = cleandoc( + """ + Checks whether invalid config is present in input file + """) + arg_parser = argparse.ArgumentParser( + prog="configCheck", + description=desc, + formatter_class=argparse.RawDescriptionHelpFormatter) + arg_parser.add_argument( + "-f", "--infile", metavar="", + help="Checks whether invalid config is present in the input file", + required=True) + arg_parser.add_argument( + "-v", "--verbose", action="store_true", help="show verbose output") + arg_parser.add_argument( + '-V', '--version', action='version', + version='%(prog)s {}'.format(__version__)) + try: + args = arg_parser.parse_args() + except IOError as e: + exit(str(e)) + # obtain logging parameters and setup logging + conf_file_path = os.path.dirname(args.infile) + conf_file_name = os.path.basename(args.infile) + check_classic_configs.check_configs_init() + new_path = os.path.join(conf_file_path, "issues_" + conf_file_name) + with open(args.infile, 'r') as infile: + with open(new_path, 'w') as outfile: + check_config_file(infile, outfile, args.verbose) + + +if __name__ == '__main__': + main() diff --git a/nspepi/nspepi2/convert_auth_cmd.py b/nspepi/nspepi2/convert_auth_cmd.py new file mode 100644 index 0000000..10ddaef --- /dev/null +++ b/nspepi/nspepi2/convert_auth_cmd.py @@ -0,0 +1,466 @@ +#!/usr/bin/env python2 + +# Copyright 2021 Citrix Systems, Inc. All rights reserved. +# Use of this software is governed by the license terms, if any, +# which accompany or are included with this software. + +import copy +import logging +from collections import OrderedDict + +import nspepi_common as common +from nspepi_parse_tree import * +from convert_classic_expr import * +import convert_cli_commands as cli_cmds + +# TODO Some of the Client Security Expressions do not have equivalent Advanced +# expressions. This may lead to some policies being converted and some not, +# which in overall will lead to invalid config. To avoid this issue, +# disabling the Classic Authentication policy and its bindings conversion for now. + +# All module names starting with "convert_" are parsed to detect and register +# class methods +@common.register_class_methods +class Authentication(cli_cmds.ConvertConfig): + """ + Converts classic Authentication policies and + authentication vserver bind commands of classic policies. + """ + + # override + bind_default_goto = "NEXT" + flow_type_direction_default = None + + def __init__(self): + """ + Information about authentication commands. + _converted_bind_cmd_trees - Dictionary to store converted + bind commands. + {: } + _policy_label_priority - Needs to add priority in policy label + bind commands. This variable contains + last priority that is used in each + policy label. + """ + self._converted_bind_cmd_trees = OrderedDict() + self._policy_label_priority = OrderedDict() + + @property + def converted_bind_cmd_trees(self): + return self._converted_bind_cmd_trees + + #@common.register_for_cmd("add", "authentication", "webAuthPolicy") + #@common.register_for_cmd("add", "authentication", "dfaPolicy") + def convert_webAuth_dfa_policy(self, auth_policy_parse_tree): + """ + Converting classic webAuth/dfa policy + to advanced authentication policy. + Syntax: + add authentication webAuthPolicy + -rule -action + or + add authentication dfaPolicy + -rule -action + converts to + add authentication Policy + -rule -action + """ + # Because we are currently converting authentication policies and its + # bindings to authentication vserver only and not VPN vserver, VPN + # global and system global bindings, we will get error for VPN + # vserver, VPN global and system global bindings. To aviod this issue, + # we create advanced policy which is equivalent to old classic policy, + # give it a name, and replace all the references to the old classic + # policy in the converted bind commands to the corresponding advanced + # policy. + original_tree = copy.deepcopy(auth_policy_parse_tree) + tree_list = [original_tree] + policy_name = auth_policy_parse_tree.positional_value(0).value + pol_obj = common.Policy(policy_name, self.__class__.__name__) + common.pols_binds.store_policy(pol_obj) + # Changing ot of the command. + auth_policy_parse_tree.ot = "Policy" + # Changing classic rule to advanced rule. + cli_cmds.ConvertConfig.convert_keyword_expr(auth_policy_parse_tree, + 'rule') + if auth_policy_parse_tree.upgraded: + self.replace_advanced_name(auth_policy_parse_tree) + tree_list.append(auth_policy_parse_tree) + pol_obj.policy_type = "classic" + else: + pol_obj.policy_type = "advanced" + return tree_list + + #@common.register_for_cmd("add", "authentication", "certPolicy") + #@common.register_for_cmd("add", "authentication", "negotiatePolicy") + #@common.register_for_cmd("add", "authentication", "tacacsPolicy") + #@common.register_for_cmd("add", "authentication", "samlPolicy") + #@common.register_for_cmd("add", "authentication", "radiusPolicy") + #@common.register_for_cmd("add", "authentication", "ldapPolicy") + #@common.register_for_cmd("add", "authentication", "localPolicy") + def convert_other_auth_policy(self, auth_policy_parse_tree): + """ + Converting local/ldap/radius/saml/tacacs/negotiate/cert policy to + advanced authentication policy. + Syntax for localPolicy: + add authentication localPolicy + converts to + add authentication Policy -rule -action local + + syntax for other policies: + add authentication + converts to + add authentication Policy -rule + -action + """ + # Because we are currently converting authentication policies and its + # bindings to authentication vserver only and not VPN vserver, VPN + # global and system global bindings, we will get error for VPN + # vserver, VPN global and system global bindings. To aviod this issue, + # we create advanced policy which is equivalent to old classic policy, + # give it a name, and replace all the references to the old classic + # policy in the converted bind commands to the corresponding advanced + # policy. + original_tree = copy.deepcopy(auth_policy_parse_tree) + tree_list = [original_tree] + policy_name = auth_policy_parse_tree.positional_value(0).value + pol_obj = common.Policy(policy_name, self.__class__.__name__) + common.pols_binds.store_policy(pol_obj) + is_local_policy = (auth_policy_parse_tree.ot.lower() == "localpolicy") + + # Changing ot of the command. + auth_policy_parse_tree.ot = "Policy" + + # Changing classic rule to advanced rule + # positional to keyword. + cli_cmds.ConvertConfig.convert_pos_expr(auth_policy_parse_tree, 1) + if not auth_policy_parse_tree.upgraded: + pol_obj.policy_type = "advanced" + return tree_list + advanced_expr = auth_policy_parse_tree.positional_value(1).value + rule_keyword = CLIKeywordParameter(CLIKeywordName("rule")) + rule_keyword.add_value(advanced_expr) + auth_policy_parse_tree.add_keyword(rule_keyword) + auth_policy_parse_tree.remove_positional(1) + + # Changing action from positional to keyword + action = None + if is_local_policy: + action_name = "LOCAL" + else: + action_name = auth_policy_parse_tree.positional_value(1).value + auth_policy_parse_tree.remove_positional(1) + action_keyword = CLIKeywordParameter(CLIKeywordName("action")) + action_keyword.add_value(action_name) + auth_policy_parse_tree.add_keyword(action_keyword) + + auth_policy_parse_tree.set_upgraded() + pol_obj.policy_type = "classic" + self.replace_advanced_name(auth_policy_parse_tree) + tree_list.append(auth_policy_parse_tree) + return tree_list + + def replace_advanced_name(self, auth_policy_parse_tree): + """ + Replace policy name with the corresponding advanced policy name and + store the policy name as classic as this is converted command. + auth_policy_parse_tree - bind command parse tree + """ + policy_name = auth_policy_parse_tree.positional_value(0).value + advanced_policy_name = "nspepi_adv_" + policy_name + auth_policy_parse_tree.positional_value(0) \ + .set_value(advanced_policy_name) + pol_obj = common.Policy(advanced_policy_name, + self.__class__.__name__, "classic") + common.pols_binds.store_policy(pol_obj) + + def convert_auth_policy_auth_vserver_bind(self, bind_cmd_parse_tree): + """ + This is a helper function which converts bind command + of authentication policy to authentication vserver. + If advanced policy is bound, returns the original tree. + If classic policy which is converted is bound + then convert the command as below: + classic policies can be bound as primary, secondary, groupExtraction. + 1. For primary policy, add -priority or update if already present. + since same priority can be given multiple times in classic policies. + Example: + bind authentication vserver -policy + converts to + bind authentication vserver -policy + -priority -gotoPriorityExpression NEXT + 2. For secondary policy, follow the below steps + 1) create authentication policylabel + _secondary_auth_label + 2) Bind all policies which are bound as + secondary to this policy label + 3) Add -nextfactor _seconday_auth_label + to all policies which are bound as primary to that bind point. + 3. For groupExtraction policy, follow the below steps + 1) create authentication policylabel + _group_auth_label + 2) Bind all policies with group_extraction + to this policy label. + 3) Add -nextfactor _group_auth_label + to all policies which has -secondary + Example: + bind authentication vserver v1 -policy p1 + bind authentication vserver v1 -policy p2 + bind authentication vserver v1 -policy p3 -secondary + bind authentication vserver v1 -policy p4 -secondary + bind authentication vserver v1 -policy p5 -groupExtraction + Converts to + add authentication policylabel v1_group_auth_label + add authentication policylabel v1_secondary_auth_label + bind authentication vserver v1 -policy p1 -priority 10 + -nextFactor v1_secondary_auth_label + -gotoPriorityExpression NEXT + bind authentication vserver v1 -policy p2 -priority 20 + -nextFactor v1_secondary_auth_label + -gotoPriorityExpression NEXT + bind authentication policylabel v1_secondary_auth_label + -policyName p3 -priority 30 -nextFactor v1_group_auth_label + -gotoPriorityExpression NEXT + bind authentication policylabel v1_secondary_auth_label + -policyName p4 -priority 40 -nextFactor v1_group_auth_label + -gotoPriorityExpression NEXT + bind authentication policylabel v1_group_auth_label + -policyName p5 -priority 30 -gotoPriorityExpression NEXT + """ + policy_name = bind_cmd_parse_tree.keyword_value("policy")[0].value + policy_type = common.pols_binds.policies[policy_name].policy_type + if policy_type == "advanced": + return [bind_cmd_parse_tree] + # Getting bind point name + vserver_name = bind_cmd_parse_tree.positional_value(0).value + bind_point = "auth_vserver_" + vserver_name + sec_auth_policy_label = vserver_name + "_secondary_auth_label" + group_factor_policy_label = vserver_name + "_group_auth_label" + + # Checking for -secondary and -groupExtraction. + # When both options are not present, it means + # policy bound is primary. + if bind_cmd_parse_tree.keyword_exists("secondary"): + # If entry for bind_point is not in dictionary, + # then add one entry. + if bind_point not in self._converted_bind_cmd_trees: + self._converted_bind_cmd_trees[bind_point] = [] + # If secondary policy label is not added already, add it. + if not self.is_policy_label_added( + bind_point, sec_auth_policy_label): + self.add_policy_label(bind_point, sec_auth_policy_label) + self.add_nextfactor_to_primary( + bind_point, sec_auth_policy_label) + # Bind policy to secondary policy label + self.bind_policy_label( + bind_point, sec_auth_policy_label, policy_name) + elif bind_cmd_parse_tree.keyword_exists("groupExtraction"): + # If entry for bind_point is not in dictionary, + # then add one entry + if bind_point not in self._converted_bind_cmd_trees: + self._converted_bind_cmd_trees[bind_point] = [] + # If group policy label is not added already, add it. + if not self.is_policy_label_added( + bind_point, group_factor_policy_label): + self.add_policy_label(bind_point, group_factor_policy_label) + self.add_nextfactor_to_secondary( + bind_point, sec_auth_policy_label, + group_factor_policy_label) + # Bind policy to group policy label + self.bind_policy_label( + bind_point, group_factor_policy_label, policy_name) + else: + if bind_point not in self._converted_bind_cmd_trees: + self._converted_bind_cmd_trees[bind_point] = [] + # Replace with advanced policy name. + advanced_policy_name = "nspepi_adv_" + policy_name + self.update_tree_arg(bind_cmd_parse_tree, "policy", + advanced_policy_name) + self._converted_bind_cmd_trees[bind_point].append( + bind_cmd_parse_tree) + + return [] + + def is_policy_label_added(self, bind_point, label_name): + """ + Checks whether policy label is added or not. + bind_point - bind_point name which is used as key in dictionary + label_name - policy label name that has to be checked for. + Returns True, if add policylabel tree is added to + _converted_bind_cmd_trees[bind_point]. + command: + add authentication policylabel + """ + cmd_list = self._converted_bind_cmd_trees[bind_point] + for cmd_parse_tree in cmd_list: + if (cmd_parse_tree.ot.lower() == "policylabel" and + cmd_parse_tree.positional_value(0).value == label_name): + return True + return False + + def add_policy_label(self, bind_point, label_name): + """ + Creates parse tree for "add authentication policylabel" command + with name label_name. + Command: + add authentication policylabel + Saves the parse tree in _converted_bind_cmd_trees dictionary. + bind_point - bind_point name which is used as key in dictionary + label_name - Authentication policy label name that has to be added. + """ + # Tree construction + pol_label_tree = CLICommand("add", "authentication", "policylabel") + pos = CLIPositionalParameter(label_name) + pol_label_tree.add_positional(pos) + # Save in dictionary + self._converted_bind_cmd_trees[bind_point].insert(0, pol_label_tree) + + def bind_policy_label(self, bind_point, policy_label, policy_name): + """ + Creates parse tree for "bind authentication policylabel" command with + given policy label and policy. + command: + bind authentication policylabel -policyName + -priority -gotoPriorityExpression NEXT + Saves the parse tree in _converted_bind_cmd_trees dictionary. + bind_point - bind_point name which is used as key in dictionary + policy_label - Policy label name to which policy has to be bound + policy_name - Policy to be bound to policy label + """ + # Tree construction + if policy_label not in self._policy_label_priority: + self._policy_label_priority[policy_label] = 0 + self._policy_label_priority[policy_label] += 100 + bind_label_tree = CLICommand("bind", "authentication", "policylabel") + pos = CLIPositionalParameter(policy_label) + bind_label_tree.add_positional(pos) + policy_key = CLIKeywordParameter(CLIKeywordName("policyName")) + advanced_policy_name = "nspepi_adv_" + policy_name + policy_key.add_value(advanced_policy_name) + bind_label_tree.add_keyword(policy_key) + priority_key = CLIKeywordParameter(CLIKeywordName("priority")) + priority_key.add_value(str(self._policy_label_priority[policy_label])) + bind_label_tree.add_keyword(priority_key) + goto_key = \ + CLIKeywordParameter(CLIKeywordName("gotoPriorityExpression")) + goto_key.add_value("NEXT") + bind_label_tree.add_keyword(goto_key) + # Save in dictionary + self._converted_bind_cmd_trees[bind_point].append(bind_label_tree) + + def add_nextfactor_to_primary(self, bind_point, label_name): + """ + Adds -nextFactor to all the policies which are + bound as primary to that bindpoint + Primary policies are bound to authentication vserver by the following + command: + bind authentication vserver -policy + bind_point - bind_point name which is used as key in dictioanry + label_name - Policy label name which should be added as nextfactor + """ + cmd_list = self._converted_bind_cmd_trees[bind_point] + for index in range(len(cmd_list)): + cmd_parse_tree = cmd_list[index] + if ((' '.join(cmd_parse_tree.get_command_type())).lower() == + "bind authentication vserver"): + self.add_nextfactor(cmd_parse_tree, label_name) + + def add_nextfactor_to_secondary( + self, bind_point, secondary_label_name, group_label_name): + """ + Adds -nextFactor to all the policies which + are bound as secondary to that bind point. + Secondary policies are bound to secondary_policy_label by the + following command: + bind authentication policylabel + -policyName + -priority + bind_point - bind point name which is used as + key in dictionary + group_label_name - Policy label name which should be + added as nextFactor + secondary_label_name - Policy label name to which nextfactor + has to be added + """ + cmd_list = self._converted_bind_cmd_trees[bind_point] + for index in range(len(cmd_list)): + cmd_parse_tree = cmd_list[index] + if ((' '.join(cmd_parse_tree.get_command_type())).lower() == + "bind authentication policylabel" and + cmd_parse_tree.positional_value(0).value == + secondary_label_name): + self.add_nextfactor(cmd_parse_tree, group_label_name) + + def add_nextfactor(self, tree, policy_label_name): + """ + Adds -nextfactor to command. + tree - command parse tree to which nextfactor + has to be added + policy_label_name - Policy label name which + should be added as nextfactor + """ + nextfactor_key = CLIKeywordParameter(CLIKeywordName("nextFactor")) + nextfactor_key.add_value(policy_label_name) + tree.add_keyword(nextfactor_key) + tree.set_upgraded() + + @common.register_for_cmd("bind", "authentication", "vserver") + def convert_auth_vserver_bind(self, bind_parse_tree): + """ + Handles Authentication vserver bind + command. + bind authentication vserver [-policy + [-priority ] [-gotoPriorityExpression + ]] + """ + if not bind_parse_tree.keyword_exists('policy'): + return [bind_parse_tree] + + policy_name = bind_parse_tree.keyword_value("policy")[0].value + priority_arg = "priority" + goto_arg = "gotoPriorityExpression" + + class_name = self.__class__.__name__ + policy_type = common.pols_binds.get_policy(policy_name).module + # If policy is Authentication policy. + if policy_type == class_name: + return self.convert_auth_policy_auth_vserver_bind( + bind_parse_tree) + + """ + Calls the method that is registered for the particular + policy type that is bound to vserver. Returns converted_list. + If the policy module is not registered for binding, + then returns the original parse tree. + """ + key = "Authentication" + if key in common.bind_table: + if policy_type in common.bind_table[key]: + m = common.bind_table[key][policy_type] + return m.method(m.obj, bind_parse_tree, policy_name, + priority_arg, goto_arg) + return [bind_parse_tree] + + @common.register_for_final_call + def get_converted_auth_bind_cmds(self): + """ + Returns all command parse trees saved in _converted_bind_cmd_trees. + This should be called only at the end of processing + of entire ns.conf file. + Return value - list of parse trees. + """ + tree_list = [] + policy_type = self.__class__.__name__ + priority_arg = "priority" + goto_arg = "gotoPriorityExpression" + for bind_point in self._converted_bind_cmd_trees: + for tree in self._converted_bind_cmd_trees[bind_point]: + if ((' '.join(tree.get_command_type())).lower() == + "bind authentication vserver"): + policy_name = tree.keyword_value("policy")[0].value + tree_list += self.convert_entity_policy_bind( + tree, tree, policy_name, + policy_type, priority_arg, goto_arg) + else: + tree_list.append(tree) + return tree_list diff --git a/nspepi/nspepi2/convert_classic_expr.py b/nspepi/nspepi2/convert_classic_expr.py new file mode 100644 index 0000000..c4814a3 --- /dev/null +++ b/nspepi/nspepi2/convert_classic_expr.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python2 + +# Copyright 2021 Citrix Systems, Inc. All rights reserved. +# Use of this software is governed by the license terms, if any, +# which accompany or are included with this software. + +import logging +import subprocess +import re +import convert_cli_commands as cli_commands +import nspepi_common as common + + +from pi_lex import PILex +from nspepi_parse_tree import CLIParseTreeNode + +eval_classic_expr = re.compile(r'SYS\s*\.\s*EVAL_CLASSIC_EXPR\s*\(\s*"', + re.IGNORECASE) + +q_s_expr = re.compile(r'\b((Q\.HOSTNAME)|(Q\.TRACKING)|' + r'(Q\.METHOD)|(Q\.URL)|(Q\.VERSION)|' + r'(Q\.CONTENT_LENGTH)|(Q\.HEADER)|' + r'(Q\.IS_VALID)|(Q\.DATE)|' + r'(Q\.COOKIE)|(Q\.BODY)|(Q\.TXID)|' + r'(Q\.CACHE_CONTROL)|(Q\.USER)|' + r'(Q\.IS_NTLM_OR_NEGOTIATE)|' + r'(Q\.FULL_HEADER)|' + r'(Q\.LB_VSERVER)|(Q\.CS_VSERVER)|' + r'(S\.VERSION)|(S\.STATUS)|' + r'(S\.STATUS_MSG)|(S\.IS_REDIRECT)|' + r'(S\.IS_INFORMATIONAL)|(S\.IS_SUCCESSFUL)|' + r'(S\.IS_CLIENT_ERROR)|(S\.IS_SERVER_ERROR)|' + r'(S\.TRACKING)|(S\.HEADER)|(S\.FULL_HEADER)|' + r'(S\.IS_VALID)|(S\.DATE)|(S\.BODY)|' + r'(S\.SET_COOKIE)|(S\.SET_COOKIE2)|' + r'(S\.CONTENT_LENGTH)|' + r'(S\.CACHE_CONTROL)|(S\.TXID)|(S\.MEDIA))\b', + re.IGNORECASE) + +def convert_classic_expr(classic_expr): + tree_obj = CLIParseTreeNode() + info_msg = 'INFO: Expression is not converted' + \ + ' - most likely it is a valid advanced expression' + warn_msg = 'WARNING: Line numbers which has ' + \ + 'more than 8191 characters length: 0' + try: + nspepi_tool_path = common.get_nspepi_tool_path() + """Error message will be in the staring of + output, whereas warning and info messages + will be present in the last.""" + nspepi_tool_output = subprocess.check_output( + ['perl', nspepi_tool_path, '-e', classic_expr], + shell=False, stderr=subprocess.STDOUT) + """ old nspepi tool adds newline character at the end + of the converted string, so remove that character.""" + nspepi_tool_output = nspepi_tool_output.rstrip() + except subprocess.CalledProcessError as exc: + # Log the command which is failing + logging.error(exc) + # Log the error message + logging.error(exc.output) + return None + if nspepi_tool_output.startswith('ERROR:'): + """old nspepi tool throws "ERROR: Expression is in blocked list + of conversion" error for vpn client security expression.""" + logging.error(nspepi_tool_output) + return None + elif nspepi_tool_output.endswith(info_msg): + """old nspepi tool didn't convert the expression, + so return input expression""" + nspepi_tool_output = classic_expr + # classic_expr is not enclosed in quotes. + nspepi_tool_output = tree_obj.normalize(nspepi_tool_output, True) + elif nspepi_tool_output.endswith(warn_msg): + logging.warning(nspepi_tool_output) + """ If expression has more than 8191 characters, old nspepi + tool gives warning message at the end of the output. + + old nspepi tool output: + WARNING: Total number of warnings due to + expressions length greater than 8191 characters: 1 + WARNING: Line numbers which has more than 8191 characters length: 0 + + Removing warning message from the output + """ + expr_end_pos = nspepi_tool_output.find("WARNING") + nspepi_tool_output = nspepi_tool_output[0:expr_end_pos] + nspepi_tool_output = nspepi_tool_output.rstrip() + + # When NSPEPI tool is used with -e option, this handles classic built-in + # Named expressions. When tool is used with -f option, all named + # expressions are handled here. + nspepi_tool_output = cli_commands.ConvertConfig.replace_named_expr( + cli_commands.remove_quotes(nspepi_tool_output)) + nspepi_tool_output = tree_obj.normalize(nspepi_tool_output, True) + return nspepi_tool_output + +def convert_adv_expr(advanced_expr): + """ + Converts Q and S prefixes. + Converts SYS.EVAL_CLASSIC_EXPR expression in advanced expressions to remove + classic expressions. + advanced_expr - Expression in which Q and S prefixes and SYS.EVAL_CLASSIC_EXPR + expression should be replaced. + Returns None in case of any Error. Otherwise returns converted expression. + """ + advanced_expr = convert_q_s_expr(advanced_expr) + return convert_sys_eval_classic_expr(advanced_expr) + +def convert_q_s_expr(advanced_expr): + """ + Convertes Q and S prefixes to use HTTP.REQ and HTTP.RES + advanced_expr - Expression in which Q and S prefixes + should be replaced. + Returns converted expression. + """ + q_s_expr_list = [] + # Get all indexes of Q and S expressions. + for match in re.finditer(q_s_expr, advanced_expr): + q_s_expr_list.append(match.start()) + for expr_index in reversed(q_s_expr_list): + if (advanced_expr[expr_index] == 'Q' or + advanced_expr[expr_index] == 'q'): + converted_expr = "HTTP.REQ" + else: + converted_expr = "HTTP.RES" + advanced_expr = (advanced_expr[0: expr_index] + + converted_expr + + advanced_expr[expr_index + 1:]) + return advanced_expr + +def convert_sys_eval_classic_expr(advanced_expr): + """ + Converts SYS.EVAL_CLASSIC_EXPR expression in advanced expressions to remove + classic expressions. + advanced_expr - Expression in which SYS.EVAL_CLASSIC_EXPR expression + should be replaced. + Returns None in case of any Error. Otherwise returns converted expression. + """ + original_expr = advanced_expr + advanced_expr_length = len(advanced_expr) + sys_eval_list = [] + # Get all indexes where SYS.EVAL_CLASSIC_EXPR starts. + for match in re.finditer(eval_classic_expr, advanced_expr): + start_index = match.start() + length = match.end() - match.start() + sys_eval_list.append([start_index, length]) + for sys_exp_info in reversed(sys_eval_list): + # arg_start_index points to opening quote in + # SYS.EVAL_CLASSIC_EXPR("<>") + sys_start_index = sys_exp_info[0] + sys_length = sys_exp_info[1] + arg_start_index = sys_start_index + sys_length - 1 + classic_exp_info = PILex.get_pi_string( + advanced_expr[arg_start_index:]) + if classic_exp_info is None: + logging.error("Error in converting expression: {}".format( + original_expr)) + return None + classic_expr = classic_exp_info[0] + length = classic_exp_info[1] + # arg_end_index points to closing quote in SYS.EVAL_CLASSIC_EXPR("<>"). + arg_end_index = arg_start_index + length - 1 + # Handle spaces between closing quote and closing brace. + sys_end_index = arg_end_index + 1 + while(sys_end_index < advanced_expr_length and + advanced_expr[sys_end_index] != ')' and + advanced_expr[sys_end_index] in " \t\r"): + sys_end_index += 1 + if (sys_end_index >= advanced_expr_length or + advanced_expr[sys_end_index] != ')'): + logging.error("Error in converting expression: {}".format( + original_expr)) + return None + converted_expr = convert_classic_expr(classic_expr) + if converted_expr is not None: + # Result from convert_classic_expr will have enclosing quotes. + converted_expr = cli_commands.remove_quotes(converted_expr) + if converted_expr is None or converted_expr == classic_expr: + logging.error("Error in converting expression: {}".format( + original_expr)) + return None + # Converted expression should be enclosed in braces because + # SYS.EVAL_CLASSIC_EXPR can have && or ||. + advanced_expr = (advanced_expr[0: sys_start_index] + '(' + + converted_expr + ')' + + advanced_expr[sys_end_index + 1:]) + tree_obj = CLIParseTreeNode() + advanced_expr = tree_obj.normalize(advanced_expr, True) + return advanced_expr diff --git a/nspepi/nspepi2/convert_cli_commands.py b/nspepi/nspepi2/convert_cli_commands.py new file mode 100644 index 0000000..94d3397 --- /dev/null +++ b/nspepi/nspepi2/convert_cli_commands.py @@ -0,0 +1,2291 @@ +#!/usr/bin/env python2 + +# Copyright 2021 Citrix Systems, Inc. All rights reserved. +# Use of this software is governed by the license terms, if any, +# which accompany or are included with this software. + +import collections +import copy + +import cli_lex +import nspepi_common as common +import convert_classic_expr +from nspepi_parse_tree import * + +# All module names starting with "convert_" are parsed to detect and register +# class methods + + +def convert_cli_init(): + """Initialize global variables uses by this module""" + global vserver_protocol_dict + vserver_protocol_dict = OrderedDict() + global policy_entities_names + global classic_entities_names + global named_expr + named_expr = {} + policy_entities_names = set() + classic_entities_names = set() + # Register built-in named expressions. + NamedExpression.register_built_in_named_exprs() + global cli_global_binds + global cli_vserver_binds + global cli_user_binds + global cli_group_binds + global cli_service_binds + cli_global_binds = OrderedDict() + cli_vserver_binds = OrderedDict() + cli_user_binds = OrderedDict() + cli_group_binds = OrderedDict() + cli_service_binds = OrderedDict() + + +def remove_quotes(val): + """ + Helper function to remove the surrounding + quotes from a CLI parameter. + val - CLI parameter that needs quotes removed. + Returns the dequoted CLI parameter + """ + result = val + if val.startswith('"') or val.startswith("'"): + lexer = cli_lex.Lexer() + lexer.input(val) + token = lexer.token() + assert token.type == "NON_KEY" + result = token.value + return result + + +def get_advanced_name(classic_name): + """ + Helper function to get a valid Advanced identifier + corresponding to a Classic identifier. + Note that this does not deal with reserved words. + Nor does it check for duplicates. + """ + adv_name = "nspepi_adv_" + re.sub(r'[^a-zA-Z0-9_]', '_', classic_name) + return adv_name + + +def get_classic_expr_list(expr): + """ + Helper function to get the list of + classic expression names present in the + given expression. + expr - Expression in which classic + expression names need to be found. + Returns the list of items found or None if none; + the items are each a list with: + - classic expression name + - advanced expression name + - start offset of token to replace + - length of token to replace + """ + lexer = cli_lex.Lexer() + lexer.input(expr) + classic_expr_info_list = [] + while True: + next_token = lexer.adv_expr_token() + if not next_token: + break + token_value = str(next_token) + token_value_len = len(token_value) + if token_value in NamedExpression.built_in_named_expr: + # Checking for built-in classic Named expression. + adv_expr_name = NamedExpression.built_in_named_expr[token_value] + else: + adv_expr_name = get_advanced_name(token_value) + if (next_token.type == "IDENTIFIER" and + adv_expr_name.lower() in policy_entities_names): + start_offset = next_token.lexpos - token_value_len + 1 + expr_info = [token_value, adv_expr_name, + start_offset, token_value_len] + classic_expr_info_list.append(expr_info) + + return classic_expr_info_list + + +class ConvertConfig(object): + """Base class to convert the config""" + + @staticmethod + def replace_named_expr(rule_expr): + """ + Helper function to replace the classic named + expression with the advanced named expression + in the given expression. + rule_expr - the expression to modify + Returns the expression with names modified as needed. + """ + converted_expr = rule_expr + for expr_info in reversed(get_classic_expr_list(rule_expr)): + # Work in reverse order to avoid recomputing offsets + offset = expr_info[2] + replace_len = expr_info[3] + converted_expr = (converted_expr[0: offset] + + expr_info[1] + converted_expr[offset + + replace_len:]) + + return converted_expr + + @staticmethod + def convert_pos_expr(commandParseTree, pos): + """ + Convert the expression present at a given position + commandParseTree - the parse tree to modify + pos - the position of the parameter to modify + Returns the modified parse tree. + """ + rule_node = commandParseTree.positional_value(pos) + rule_expr = rule_node.value + converted_expr = convert_classic_expr.convert_classic_expr(rule_expr) + if converted_expr is None: + logging.error('Error in converting command : ' + + str(commandParseTree)) + converted_expr = rule_expr + else: + # converted_expr will have quotes and rule_expr will not have + # quotes. Since we are comparing these 2 expressions, removing + # quotes from converted_expr. + converted_expr = remove_quotes(converted_expr) + if converted_expr != rule_expr: + # expression is converted, this is classic. + rule_node.set_value(converted_expr) + commandParseTree.set_upgraded() + else: + # expression is not converted, then it can be advanced + # expression. Advanced expressions can have Q and S prefixes and + # SYS.EVAL_CLASSIC_EXPR expression which needs to be converted. + commandParseTree = ConvertConfig \ + .convert_adv_expr_list(commandParseTree, [pos]) + return commandParseTree + + @staticmethod + def convert_keyword_expr(commandParseTree, keywordName): + """ + Convert the expression present as a value of + the given keyword name. + commandParseTree - the parse tree to modify + keywordName - the name of the keyword parameter to modify + Returns the modified parse tree. + """ + rule_node = commandParseTree.keyword_value(keywordName) + rule_expr = rule_node[0].value + converted_expr = convert_classic_expr.convert_classic_expr(rule_expr) + if converted_expr is None: + logging.error('Error in converting command : ' + + str(commandParseTree)) + converted_expr = rule_expr + else: + # converted_expr will have quotes and rule_expr will not have + # quotes. Since we are comparing these 2 expressions, removing + # quotes from converted_expr. + converted_expr = remove_quotes(converted_expr) + if converted_expr != rule_expr: + # expression is converted, this is classic. + rule_node[0].set_value(converted_expr) + commandParseTree.set_upgraded() + else: + # expression is not converted, then it can be advanced + # expression. Advanced expressions can have Q and S prefixes and + # SYS.EVAL_CLASSIC_EXPR expression which needs to be converted. + commandParseTree = ConvertConfig \ + .convert_adv_expr_list(commandParseTree, [keywordName]) + return commandParseTree + + @staticmethod + def convert_adv_expr_list(tree, param_list): + """ + Converts Q and S prefixes and SYS.EVAL_CLASSIC_EXPR expression from the given + list of parameters. + tree - the parse tree to modify + param_list - list of parameters to modify. Each Parameter can be either + positional parameter or keyword parameter. + If its a keyword parameter, mention the keyword name. + If its a positional parameter, mention the position of + the parameter. + Returns the modified parse tree. + """ + original_tree = copy.deepcopy(tree) + for param in param_list: + adv_expr = common.get_cmd_arg(param, tree) + if adv_expr is None: + continue + converted_expr = convert_classic_expr.convert_adv_expr(adv_expr) + if converted_expr is None: + logging.error('Error in converting command : ' + + str(original_tree)) + return original_tree + else: + converted_expr = remove_quotes(converted_expr) + if converted_expr != adv_expr: + if isinstance(param, int): + # Positional Parameter + tree.positional_value(param).set_value(converted_expr) + else: + # Keyword Parameter + tree.keyword_value(param)[0].set_value(converted_expr) + tree.set_adv_upgraded() + return tree + + def store_builtin_policies(self): + """ + Creates and stores Policy object for built-in policies. + """ + # Since built-in policy add commands are not saved + # in ns.conf, function registered for add commands will + # not be called for built-in policies where policy object + # is stored. + for policy_name in self.built_in_policies: + pol_obj = common.Policy(policy_name, self.__class__.__name__, + "classic") + common.pols_binds.store_policy(pol_obj) + pol_obj = common.Policy(self.built_in_policies[policy_name], + self.__class__.__name__, "advanced") + common.pols_binds.store_policy(pol_obj) + + @staticmethod + def register_policy_entity_name(commandParseTree): + """ Add the entity name in the global list.""" + name = commandParseTree.positional_value(0).value.lower() + policy_entities_names.add(name) + + @staticmethod + def register_classic_entity_name(commandParseTree): + """ Add the classic entity name in the classic global list.""" + name = commandParseTree.positional_value(0).value.lower() + classic_entities_names.add(name) + + """ + POLICY BIND CONVERSION INFRASTRUCTURE + Converting binds of classic policies to the equivalent advanced + policies has a number of issues to address: + + 1. There can be multiple classic policies bound to same priority + and classic policies can be bound without priority also. + The equivalent advanced policies must be bound at separate + policies that maintain the original order. + + 2. Binds for policies for classic modules that are being replaced + by an advanced module (e.g. Filter by Rewrite and Responder) + may need to be inserted before or after existing policy binds + to maintain the same order of evaluation. + + Use of this infrastructure for a module + + 1. Each module class must be a subclass of ConvertConfig to inherit + the infrastructure methods and attributes. + + 2. If the bind command does not have gotoPriorityExpression, + then it uses the default gotoPriorityExpression of END. + If the module class does not want to use the default value, then + the module class must override the bind_default_goto attribute. + If the module class does not want to add gotoPriorityExpression + to bind command, then override the bind_default_goto attribute + with None. + + 3. If priority and gotopriorityexpression are positional arguments + in the bind command and if the bind command does not have these values + initially and the binding infra is used to add a priority and default + goto value then, priority and gotopriorityexpression will + be added at the end of the positional arguments list. + example: + bind global + converts to + bind global + + 4. Each parsed bind command is processed by either: + + .convert_global_bind(parse_tree, module, priority_arg, + goto_arg, position) + or + .convert_entity_policy_bind(parse_tree, + policy_module, priority_arg, goto_arg, position) + + + These methods save bind commands in the appropriate dictionaries. + These methods can be used for bindings of policies to + global/vserver/user/group/service. + """ + + class BindInfo(object): + """ + Object to hold the bind command info. + """ + def __init__(self): + """ + Bind command information. + orig_cmd - original bind command read from config. + parse_tree - bind command parse tree. + position - position where the bind command + has to be inserted. Possible + values - before, inplace, after. + priority - priority value. + goto - gotopriorityexpression value. + bind_arg_priority - Provides the positional index + or keyword name for the priority + parameter. + bind_arg_goto - Provides the positional index or + keyword name for the goto parameter. + policy_type - type of policy("classic" or "advanced") + flow_type_direction - bind type information("REQUEST" or + "RESPONSE") + """ + self.orig_cmd = "" + self.parse_tree = None + self.position = "" # "before", "inplace", or "after" insertion + self.priority = 0 + self.goto = "" + self.bind_arg_priority = "priority" + self.bind_arg_goto = "gotoPriorityExpression" + self.policy_type = None + self.flow_type_direction = None + + def set(self, orig_cmd, parse_tree, position, priority, goto, + priority_arg, goto_arg, policy_type, flow_type_direction): + """ + Sets the BindInfo class instance variables. + orig_cmd - original bind command read from config. + parse_tree - bind comman parse tree. + position - position of insertion. + priority - priority + goto - gotopriorityexpression value. + priority_arg - Positional index or keyword name + for the priority argument. + goto_arg - positional index or keyword name + for the goto argument. + flow_type_direction - bind type information( "REQUEST" + or "RESPONSE") + """ + self.orig_cmd = orig_cmd + self.parse_tree = parse_tree + self.position = position + self.priority = priority + self.goto = goto + self.bind_arg_priority = priority_arg + self.bind_arg_goto = goto_arg + self.policy_type = policy_type + self.flow_type_direction = flow_type_direction + + def get_bind_dict(self, current_dict, key): + """ + Return the dictionary selected by key in current_dict. If + this dictionary does not yet exist, it will be created. + current_dict - dictionary + key - key name + """ + if key not in current_dict: + current_dict[key] = OrderedDict() + return current_dict[key] + + def save_bind_for_reprioritization_common(self, bind_dict, orig_tree, tree, + position, priority, goto, + bind_type, priority_arg, + goto_arg, policy_type): + """ + Save a bind command in the relevant dictionary for later processing + bind_dict - is a dictionary which has list of + BindInfo objects for commands for a bindpoint + (e.g. GLOBAL REWRITE REQ_DEFAULT). A BindInfo + object for the current command will be appended + to the list. + orig_tree - parsed command tree of bind command read from config. + tree - processed and possibly modified command tree of bind command. + position - indicates where the bind is to be inserted: + "before", "inplace", or "after". + priority - is the bind prority as an int; may be 0. + goto - is the gotoPriorityExpression for the bind + bind_type - is the type arg e.g. "REQ_DEFAULT" + priority_arg - Positional index or keyword name + for the priority argument. + goto_arg - Positional index or keyword name + for the goto argument. + """ + # bind_type may be None in some cases. + # bind_type is used as key for dictionary, + # so making the bind_type equal to empty string + # when the bind_type is None. + if bind_type is None: + bind_type = "" + else: + bind_type = bind_type.lower() + if bind_type not in bind_dict: + bind_dict[bind_type] = [] + bind_info = self.BindInfo() + flow_type_direction = self.flow_type_direction_default + bind_info.set(orig_tree.original_line, tree, position, int(priority), + goto, priority_arg, goto_arg, + policy_type, flow_type_direction) + bind_dict[bind_type].append(bind_info) + + def update_tree_arg(self, tree, arg, value): + """ + Modifies the parse tree argument. If arg is string it + is a keyword. If arg is an int it is a positional index. + For positional index: + Updates the positional value if the positional value + already exists. + If positional argument arg is not present, then + adds positional argument with value. + For keyword argument: + Updated the keyword value, if keyword already exists + else adds a keyword to tree with keyword name arg and + keyword value value + If value is None, argument is not added. + tree - Command parse tree. + arg - Command argument, Can be positional or keyword. + value - command argument value. + """ + if value is None: + return + if isinstance(arg, int): + if tree.positional_value(arg) is not None: + tree.positional_value(arg).set_value(value) + else: + pos = CLIPositionalParameter(value) + tree.add_positional(pos) + else: + if tree.keyword_exists(arg): + tree.keyword_value(arg)[0].set_value(value) + else: + keyword_arg = CLIKeywordParameter(CLIKeywordName(arg)) + keyword_arg.add_value(value) + tree.add_keyword(keyword_arg) + tree.set_upgraded() + + """ + The gotoPriorityExpression argument to use for converted bindings. + Override in a module class if different. + If gotoPriorityExpression argument is not required, then override + with None. + """ + bind_default_goto = "END" + + """ + The bind type side(REQUEST or RESPONSE) information used to + add -type keyword in converted bindings. + Override in a module class if different. + If -type keyword is not required, then override with None. + """ + flow_type_direction_default = "REQUEST" + + def convert_global_bind(self, orig_tree, tree, policy_name, module, + priority_arg, goto_arg, position="inplace"): + """ + Process a global bind command represented by + the command parse tree and saves the required info: + bind global + Save the bind command in the cli_global_binds dictionary. Return empty + list to delete the command. It will later be emitted after + reprioritization. + The dictionary path: + cli_global_binds[][] + Args: + orig_tree - bind command parse tree of original command from ns.conf. + In case bind command is created newly then this argument + should contain the bind command parse tree of original + command from which this new bind command is getting + created + tree - bind command parse tree. + policy_name - name of the policy that is bound in this bind command + module - Name of the policy module(e.g. appfw, tunnel) + priority_arg - identification of the parameter in the bind command + for priority + goto_arg - identification of the parameter in the bind command for goto + position - position to be inserted. + """ + if position not in ("before", "inplace", "after"): + logging.critical("unexpected insert position value") + sys.exit() + priority, goto, bind_type = self.get_common_info(tree, + priority_arg, + goto_arg) + # get policy type + policy_type = None + if policy_name in common.pols_binds.policies: + policy_type = common.pols_binds.policies[policy_name].policy_type + bind_dict = self.get_bind_dict(cli_global_binds, module.lower()) + self.save_bind_for_reprioritization_common(bind_dict, orig_tree, tree, + position, priority, goto, + bind_type, priority_arg, + goto_arg, policy_type) + # store original bind command for analysis + common.pols_binds.store_original_bind( + common.Bind( + "global", orig_tree.ot.lower(), None, policy_name, + module.lower(), bind_type if bind_type else "", str(priority), + orig_tree.original_line, lineno=orig_tree.lineno)) + return [] + + def convert_entity_policy_bind(self, orig_tree, tree, policy_name, + policy_module, priority_arg, goto_arg, + position="inplace"): + """ + Process a vserver/user/group/service bind command + represented by the command parse tree: + bind vserver + bind user + bind group + bind service + Save the bind command in the cli_vserver_binds/cli_user_binds/ + cli_group_binds/cli_service_binds dictionary. Return an empty + parse tree to delete the command. It will later be emitted after + reprioritization. + The dictionary path: + cli_vserver_binds[][][][] + cli_user_binds[][][][] + cli_group_binds[][][][] + cli_service_binds[][][][] + orig_tree - bind command parse tree of original command from ns.conf. + In case bind command is created newly then this argument + should contain the bind command parse tree of original + command from which this new bind command is getting + created + tree - bind command parse tree. + policy_name - name of the policy that is bound in this bind command + policy_module - module name of policy which is + bound to the bind command. + priority_arg - identification of the parameter in the bind command + for priority + goto_arg - identification of the parameter in the bind command for goto + position - position to be inserted. + """ + if position not in ("before", "inplace", "after"): + logging.critical("unexpected insert position value") + sys.exit() + entity = tree.ot.lower() + entity_type = tree.group + entity_name = common.get_cmd_arg(0, tree) + priority, goto, bind_type = self.get_common_info(tree, priority_arg, + goto_arg) + policy_type = None + # get policy type + if policy_name in common.pols_binds.policies: + policy_type = common.pols_binds.policies[policy_name].policy_type + if entity == "vserver": + bind_dict = self.get_bind_dict(cli_vserver_binds, + entity_type.lower()) + elif entity == "user": + bind_dict = self.get_bind_dict(cli_user_binds, entity_type.lower()) + elif entity == "group": + bind_dict = self.get_bind_dict(cli_group_binds, + entity_type.lower()) + elif entity == "service": + bind_dict = self.get_bind_dict(cli_service_binds, + entity_type.lower()) + else: + logging.critical("Unexpected command " + str(tree)) + sys.exit() + bind_dict = self.get_bind_dict(bind_dict, entity_name) + bind_dict = self.get_bind_dict(bind_dict, policy_module.lower()) + self.save_bind_for_reprioritization_common(bind_dict, orig_tree, tree, + position, priority, goto, + bind_type, priority_arg, + goto_arg, policy_type) + # store original bind command for analysis + common.pols_binds.store_original_bind( + common.Bind( + entity, entity_type.lower(), entity_name, policy_name, + policy_module.lower(), (bind_type if bind_type else ""), + str(priority), orig_tree.original_line, + lineno=orig_tree.lineno)) + return [] + + def get_common_info(self, tree, priority_arg, goto_arg): + """ + Returns priority, gotoPriorityExpression and + bind type value from the parse tree. + tree - parse tree + """ + priority = common.get_cmd_arg(priority_arg, tree) + if priority is None: + priority = 0 + # Add priority argument to tree. + self.update_tree_arg(tree, priority_arg, str(priority)) + else: + priority = int(priority) + + goto = common.get_cmd_arg(goto_arg, tree) + if goto is None: + if self.bind_default_goto is not None: + goto = self.bind_default_goto + # Add goto argument to tree. + self.update_tree_arg(tree, goto_arg, str(goto)) + + bind_type = common.get_cmd_arg("type", tree) + if bind_type is not None: + if bind_type.lower() in ["req_default", "req_override"]: + bind_type = "REQUEST" + elif bind_type.lower() in ["res_default", "res_override"]: + bind_type = "RESPONSE" + return priority, goto, bind_type + + """ + Increment used when renumbering bind priorities. + """ + PRIORITY_INCREMENT = 100 + + def reprioritize_binds(self, binds): + """ + Sort the binds for the bindpoint and if necessary renumber their + priorities. + - binds is a list of BindInfo objects + Return a list of reprioritized BindInfo objects. + """ + # Sort the binds by their positions. + new_binds = [] + for position in ("before", "inplace", "after"): + for bind_info in binds: + if bind_info.position == position: + new_binds.append(bind_info) + + # Check if the bind priorities are not in the required order and so + # require renumbering. Also 0 priorities require renumbering. + need_pri_renum = False + for i in range(len(new_binds)): + if (new_binds[i].priority == 0) or ((i > 0) and + (new_binds[i-1].priority + >= new_binds[i].priority)): + need_pri_renum = True + break + + if not need_pri_renum: + return new_binds + + # Keep track of which old priority is mapped to which new priority, + # for use in changing gotoPriorityExpressions. This needs to be done + # separately for the before, inplace, and after binds, since there + # can be duplicates priorities across these groups. + old_to_new_pri = {} + for position in ("before", "inplace", "after"): + old_to_new_pri[position] = {} + new_pri = self.PRIORITY_INCREMENT + + # Renumber the priorities in the binds. + for bind_info in new_binds: + old_to_new_pri[bind_info.position][bind_info.priority] = new_pri + # Update priority in parse tree. + self.update_tree_arg(bind_info.parse_tree, + bind_info.bind_arg_priority, str(new_pri)) + new_pri += self.PRIORITY_INCREMENT + + # Check if any of the bind gotoPriorityExpressions have a priority + # that needs to be modified. Also issue a error if any + # gotoPriorityExpressions uses an expression. + for bind_info in new_binds: + goto = bind_info.goto + if goto is None: + continue + elif goto.isdigit(): + old_goto = int(goto) + max_goto = max(old_to_new_pri[bind_info.position]) + if old_goto in old_to_new_pri[bind_info.position]: + new_goto = str(old_to_new_pri[bind_info.position] + [old_goto]) + elif old_goto > max_goto: + new_goto = "END" + else: + new_goto = "1" + # Update goto in parse tree. + self.update_tree_arg(bind_info.parse_tree, + bind_info.bind_arg_goto, new_goto) + elif goto not in ("NEXT", "END", "USE_INVOCATION_RESULT"): + logging.error("gotoPriorityExpression in {} uses an" + " expression. Since the priorities for this" + " bindpoint have been renumbered, this" + " expression will need to be modified manually." + "".format(str(bind_info.parse_tree))) + return new_binds + + def reprioritize_and_emit_global_binds(self): + """ + Renumber the priorities for policy binds to all global bindpoints + and return a list of the command strings for those binds. + """ + bind_cmd_trees = [] + for module in cli_global_binds: + module_bind_dict = cli_global_binds[module] + for bind_type in module_bind_dict: + binds = module_bind_dict[bind_type] + new_binds = self.reprioritize_binds(binds) + for bind_info in new_binds: + if common.pols_binds.is_bind_unsupported( + bind_info.orig_cmd): + logging.error( + "Bind command [{}] is commented out because it" + " can't be converted to be under a valid advanced" + " bindpoint as priority needs to be changed" + " manually. However, the command is partially" + " converted as [{}]. If the command is required" + " please take a backup because comments are not" + " saved in ns.conf after triggering" + "'save ns config'.{}" + "".format(bind_info.orig_cmd.strip(), + str(bind_info.parse_tree).strip(), + common.CMD_MOD_ERR_MSG)) + bind_cmd_trees.append( + "# {}".format(str(bind_info.parse_tree))) + else: + type_key_value = None + # Combine bind type side information and global_type + # information to determine the type keyword value. + # Possible values - REQ_DEFAULT, REQ_OVERRIDE, + # RES_DEFAULT, RES_OVERRIDE. + global_type = ( + common.pols_binds.get_global_type_for_bind( + bind_info.orig_cmd)) + if (bind_info.flow_type_direction and global_type and + bind_info.flow_type_direction in + ("REQUEST", "RESPONSE") and + bind_info.policy_type == "classic"): + type_key_value = ( + bind_info.flow_type_direction[0:3] + + '_' + global_type) + self.update_tree_arg( + bind_info.parse_tree, "type", + type_key_value.upper()) + if (module == "rewrite" and + bind_info.policy_type == "classic"): + type_key_value = ( + bind_type[0:3] + '_' + global_type) + self.update_tree_arg( + bind_info.parse_tree, "type", + type_key_value.upper()) + bind_cmd_trees.append(bind_info.parse_tree) + return bind_cmd_trees + + def reprioritize_and_emit_4_level_dict(self, bind_dict): + """ + Renumber the priorities for all policy binds in bind_dict + and return a list of the command strings for those binds. + bind_dict - dictionary in which parse trees are saved. + dictionary path: + bind_dict[][][][] + """ + bind_cmd_trees = [] + for entity_type in bind_dict: + entity_type_bind_dict = bind_dict[entity_type] + for entity_name in entity_type_bind_dict: + entity_name_bind_dict = entity_type_bind_dict[entity_name] + for module in entity_name_bind_dict: + module_bind_dict = entity_name_bind_dict[module] + for bind_type in module_bind_dict: + binds = module_bind_dict[bind_type] + new_binds = self.reprioritize_binds(binds) + for bind_info in new_binds: + if common.pols_binds.is_bind_unsupported( + bind_info.orig_cmd): + logging.error( + "Bind command [{}] is commented out" + " because it can't be converted to be" + " under a valid advanced bindpoint as" + " priority needs to be changed manually." + " However, the command is partially" + " converted as [{}]. If the command is" + " required please take a backup because" + " comments are not saved in ns.conf" + " after triggering 'save ns config'." + "{}".format( + bind_info.orig_cmd.strip(), + str(bind_info.parse_tree).strip(), + common.CMD_MOD_ERR_MSG)) + bind_cmd_trees.append( + "# {}".format(str(bind_info.parse_tree))) + else: + if (bind_info.flow_type_direction and + bind_info.policy_type == "classic"): + self.update_tree_arg( + bind_info.parse_tree, "type", + bind_info.flow_type_direction.upper()) + bind_cmd_trees.append(bind_info.parse_tree) + return bind_cmd_trees + + def reprioritize_and_emit_binds(self): + """ + Renumber the priorities for all policy binds + and return a list of the command strings for those binds. + """ + bind_cmd_trees = [] + bind_cmd_trees += self.reprioritize_and_emit_global_binds() + bind_cmd_trees += \ + self.reprioritize_and_emit_4_level_dict(cli_vserver_binds) + bind_cmd_trees += \ + self.reprioritize_and_emit_4_level_dict(cli_user_binds) + bind_cmd_trees += \ + self.reprioritize_and_emit_4_level_dict(cli_group_binds) + bind_cmd_trees += \ + self.reprioritize_and_emit_4_level_dict(cli_service_binds) + return bind_cmd_trees + + +@common.register_class_methods +class CacheRedirection(ConvertConfig): + """ Handle CR feature """ + + # Classic built-in policy names and there corresponding + # advanced built-in policy names. + built_in_policies = { + "bypass-non-get": "bypass-non-get-adv", + "bypass-cache-control": "bypass-cache-control-adv", + "bypass-dynamic-url": "bypass-dynamic-url-adv", + "bypass-urltokens": "bypass-urltokens-adv", + "bypass-cookie": "bypass-cookie-adv" + } + + @common.register_for_init_call + def store_builtin_cr_policies(self): + """ + Creates and stores Policy object for built-in CR policies. + """ + self.store_builtin_policies() + + @common.register_for_cmd("add", "cr", "vserver") + def convert_cr_vserver(self, commandParseTree): + """ + Get vserver protocol to help in filter bind conversion + cr_protocol - cr vserver protocol + crv_name - cr vserver name + vserver_protocol_dict - dict to store protocol as value to the + vserver name as key + """ + cr_protocol = commandParseTree.positional_value(1).value + crv_name = commandParseTree.positional_value(0).value + vserver_protocol_dict[crv_name] = cr_protocol.upper() + return [commandParseTree] + + @common.register_for_cmd("add", "cr", "policy") + def convert_policy(self, commandParseTree): + """ + Converts classic cr policy to advanced. + """ + policy_name = commandParseTree.positional_value(0).value + pol_obj = common.Policy(policy_name, self.__class__.__name__) + common.pols_binds.store_policy(pol_obj) + """Action can be set only with advance expression, + so, only check for Q and S prefixes and sys.eval_classic_expr in + the rule. If action field is not set, then convert the rule + and set ORIGIN action.""" + if commandParseTree.keyword_exists('action'): + commandParseTree = CacheRedirection \ + .convert_adv_expr_list(commandParseTree, ["rule"]) + return [commandParseTree] + + """Convert classic CR policy to advanced. + Syntax: + add cr policy -rule + to + add cr policy -rule -action ORIGIN""" + commandParseTree = CacheRedirection \ + .convert_keyword_expr(commandParseTree, 'rule') + pol_obj.policy_type = ("classic" + if commandParseTree.upgraded else "advanced") + """If expression successfully converted into advanced, + then only add action.""" + if commandParseTree.upgraded: + action_node = CLIKeywordName('action') + action_param = CLIKeywordParameter(action_node) + action_param.add_value('ORIGIN') + commandParseTree.add_keyword(action_param) + return [commandParseTree] + + @common.register_for_cmd("bind", "cr", "vserver") + def convert_cr_vserver_bind(self, bind_parse_tree): + """ + Handles CR vserver bind command. + bind cr vserver -policyName + -priority -gotoPriorityExpression + """ + if not bind_parse_tree.keyword_exists('policyName'): + return [bind_parse_tree] + + policy_name = bind_parse_tree.keyword_value("policyName")[0].value + priority_arg = "priority" + goto_arg = "gotoPriorityExpression" + + class_name = self.__class__.__name__ + policy_type = common.pols_binds.get_policy(policy_name).module + # When policy is CR policy. + if policy_type == class_name: + # check for classic built-in policy. + if policy_name in self.built_in_policies: + self.update_tree_arg(bind_parse_tree, "policyName", + self.built_in_policies[policy_name]) + return self.convert_entity_policy_bind( + bind_parse_tree, bind_parse_tree, policy_name, + policy_type, priority_arg, goto_arg) + + """ + Calls the method that is registered for the particular + policy type that is bound to CR. Returns converted_list. + If the policy module is not registered for binding, + then returns the original parse tree. + """ + key = "CacheRedirection" + if key in common.bind_table: + if policy_type in common.bind_table[key]: + m = common.bind_table[key][policy_type] + return m.method(m.obj, bind_parse_tree, policy_name, + priority_arg, goto_arg) + + return [bind_parse_tree] + + +# TODO File based Classic Expressions do not have equivalent Advanced +# expressions. File based Classic Expressions can be used in Authorization +# policies. This may lead to some policies being converted and some not, +# which in overall will lead to invalid config. To avoid this issue, +# disabling the Classic Authorization policy and its bindings +# conversion for now. + +# TODO The Advanced Authorization policies can have Q and S prefixes and +# SYS.EVAL_CLASSIC_EXPR expression which needs to be converted. +# Registering the authorization policy to convert_advanced_expr in AdvExpression +# class to support Advance expression conversion. While enabling back the Classic +# Authorization policy conversion, remove the entry from convert_advanced_expr. +#@common.register_class_methods +class Authorization(ConvertConfig): + """ Handle Authorization feature """ + + @common.register_for_cmd("add", "authorization", "policy") + def convert_policy(self, commandParseTree): + """Convert classic Authorization policy to advanced + Syntax: + add authorization policy + to + add authorization policy + """ + policy_name = commandParseTree.positional_value(0).value + pol_obj = common.Policy(policy_name, self.__class__.__name__) + common.pols_binds.store_policy(pol_obj) + commandParseTree = Authorization.convert_pos_expr(commandParseTree, 1) + pol_obj.policy_type = ("classic" + if commandParseTree.upgraded else "advanced") + return [commandParseTree] + + # TODO need to integrate with priority interleaving. + @common.register_for_bind(["User", "Group", "LB", "ContentSwitching"]) + def convert_authz_entity_bind(self, commandParseTree, policy_name, + priority_arg, goto_arg): + """ + Handles binding of authorization policy to AAA user and group. + Arguments: + commandParseTree - bind command parse tree. + priority_arg - Indicates whether priority argument is + keyword or positional. It will be either + positional index or keyword name. + goto_arg - Indicates whether gotoPriorityExpression + argument is keyword or positional argument. + It will be either positional index or keyword name. + Returns converted list of parse trees. + """ + policy_type = self.__class__.__name__ + return self.convert_entity_policy_bind( + commandParseTree, commandParseTree, + policy_name, policy_type, priority_arg, goto_arg) + + +@common.register_class_methods +class TMSession(ConvertConfig): + """ Handle TM feature """ + + # classic built-in policy and its corresponding advanced built-in policy. + built_in_policies = { + "SETTMSESSPARAMS_POL": "SETTMSESSPARAMS_ADV_POL" + } + + def __init__(self): + """ + Adds the module to the list to skip the global + override during the priority analysis. + """ + common.PoliciesAndBinds.add_to_skip_global_override( + self.__class__.__name__.lower()) + + @common.register_for_init_call + def store_builtin_tmsession_policy(self): + """ + Creates and stores Policy object for built-in TM session policy. + """ + self.store_builtin_policies() + + @common.register_for_cmd("add", "tm", "sessionPolicy") + def convert_policy(self, commandParseTree): + """Convert classic TM session policy to advanced + Syntax: + add tm sessionPolicy + to + add tm sessionPolicy + """ + policy_name = commandParseTree.positional_value(0).value + pol_obj = common.Policy(policy_name, self.__class__.__name__) + common.pols_binds.store_policy(pol_obj) + commandParseTree = TMSession.convert_pos_expr(commandParseTree, 1) + pol_obj.policy_type = ("classic" + if commandParseTree.upgraded else "advanced") + return [commandParseTree] + + # override + flow_type_direction_default = None + bind_default_goto = "NEXT" + + @common.register_for_bind(["User", "Group", "Authentication"]) + def convert_tmsession_entity_bind(self, commandParseTree, policy_name, + priority_arg, goto_arg): + """ + Handles binding of tmsession policy to AAA user, AAA group + and Authentication vserver. + Arguments: + commandParseTree - bind command parse tree. + priority_arg - Indicates whether priority argument is + keyword or positional. It will be either + positional index or keyword name. + goto_arg - Indicates whether gotoPriorityExpression + argument is keyword or positional argument. + It will be either positional index or keyword name. + Returns converted list of parse trees. + """ + policy_type = self.__class__.__name__ + # check for classic built-in policy. + # TM session policy can be bound to aaa user, aaa group and + # Authentication vserver.In all these bind commands, keyword used + # for policy is "policy" + if policy_name in self.built_in_policies: + self.update_tree_arg(commandParseTree, "policy", + self.built_in_policies[policy_name]) + return self.convert_entity_policy_bind( + commandParseTree, commandParseTree, + policy_name, policy_type, priority_arg, goto_arg) + + @common.register_for_cmd("bind", "tm", "global") + def convert_tm_global(self, commandParseTree): + """ + Handles TM global bind command. + bind tm global [-policyName + [-priority ] [-gotoPriorityExpression + ]] + """ + policy_name = commandParseTree.keyword_value("policyName")[0].value + # TM global can bind TM sesssion policies + # and TM traffic policies. Only TM session + # policies have to be handled. + if (common.pols_binds.get_policy(policy_name).module + != self.__class__.__name__): + return [commandParseTree] + priority_arg = "priority" + goto_arg = "gotoPriorityExpression" + module = self.__class__.__name__ + # check for classic built-in policy + if policy_name in self.built_in_policies: + self.update_tree_arg(commandParseTree, "policyName", + self.built_in_policies[policy_name]) + return self.convert_global_bind(commandParseTree, + commandParseTree, policy_name, module, + priority_arg, goto_arg) + + +@common.register_class_methods +class TunnelTraffic(ConvertConfig): + """ Handle Tunnel Traffic feature """ + + flow_type_direction_default = None + + @common.register_for_cmd("add", "tunnel", "trafficPolicy") + def convert_policy(self, commandParseTree): + """Convert classic Tunnel traffic policy to advanced + Syntax: + add tunnel trafficPolicy + to + add tunnel trafficPolicy + """ + policy_name = commandParseTree.positional_value(0).value + pol_obj = common.Policy(policy_name, self.__class__.__name__) + common.pols_binds.store_policy(pol_obj) + commandParseTree = TunnelTraffic.convert_pos_expr(commandParseTree, 1) + pol_obj.policy_type = ("classic" + if commandParseTree.upgraded else "advanced") + return [commandParseTree] + + @common.register_for_cmd("bind", "tunnel", "global") + def convert_tunnel_global(self, commandParseTree): + """ + Handles tunnel global bind command. + Syntax: + bind tunnel global ( [-priority ]) + -state(ENABLED | DISABLED) [-gotoPriorityExpression ] + """ + if (commandParseTree.keyword_exists("state") and + commandParseTree.keyword_value("state")[0].value.lower() + == "disabled"): + logging.warning("Following bind command is commented out because" + " state is disabled. If command is required" + " please take a backup because comments will" + " not be saved in ns.conf after triggering" + " 'save ns config': {}" + "".format(str(commandParseTree).strip())) + return ['#' + str(commandParseTree)] + + # Classic built-in policy bindings should be disabled + # and the corresponding advanced built-in policy bindings should + # be added. + built_in_policies = { + "ns_tunnel_nocmp": "ns_adv_tunnel_nocmp", + "ns_tunnel_cmpall_gzip": "ns_adv_tunnel_cmpall_gzip", + "ns_tunnel_mimetext": "ns_adv_tunnel_mimetext", + "ns_tunnel_msdocs": "ns_adv_tunnel_msdocs" + } + policy_node = commandParseTree.positional_value(0) + policy_name = policy_node.value + disabled_bind_list = [] + if policy_name in built_in_policies: + disabled_classic_built_in_bind = copy.deepcopy(commandParseTree) + # Add -state DISABLED keyword to classic + # built-in policy bind command. + self.update_tree_arg(disabled_classic_built_in_bind, + "state", "DISABLED") + disabled_bind_list = [disabled_classic_built_in_bind] + disabled_classic_built_in_bind.set_upgraded() + # Advanced built-in policy binding. + policy_name = built_in_policies[policy_name] + policy_node.set_value(policy_name) + # Since built-in policy add commands are not saved + # in ns.conf, function registered for add commands will + # not be called for built-in policies where policy object + # is stored. + # Policy object should be stored here for built-in policies. + pol_obj = common.Policy(policy_name, self.__class__.__name__, + "classic") + common.pols_binds.store_policy(pol_obj) + commandParseTree.set_upgraded() + + priority_arg = "priority" + goto_arg = "gotoPriorityExpression" + module = self.__class__.__name__ + return disabled_bind_list + self.convert_global_bind( + commandParseTree, + commandParseTree, policy_name, + module, priority_arg, goto_arg) + +# TODO Some of the Client Security Expressions do not have equivalent Advanced +# expressions. This may lead to some policies being converted and some not, +# which in overall will lead to invalid config. To avoid this issue, +# disabling the Classic VPNTraffic policy and its bindings conversion +# for now. + +# TODO The Advanced VPNTraffic policies can have Q and S prefixes and +# SYS.EVAL_CLASSIC_EXPR expression which needs to be converted. +# Registering the VPNTraffic policy to convert_advanced_expr in AdvExpression +# class to support Advance expression conversion. While enabling back the Classic +# VPNTraffic policy conversion, remove the entry from convert_advanced_expr. + +# TODO VPN class handles bindings of VPNTraffic policies. While enabling the +# VPNTraffic policy conversion, enable VPN class as well. +#@common.register_class_methods +class VPNTraffic(ConvertConfig): + """ Handle VPN Traffic feature """ + + # override + flow_type_direction_default = None + + def __init__(self): + """ + Adds the module to the list to skip the global + override during the priority analysis. + """ + common.PoliciesAndBinds.add_to_skip_global_override( + self.__class__.__name__.lower()) + + @common.register_for_cmd("add", "vpn", "trafficPolicy") + def convert_policy(self, commandParseTree): + """Convert classic VPN traffic policy to advanced + Syntax: + add vpn trafficPolicy + to + add vpn trafficPolicy + """ + policy_name = commandParseTree.positional_value(0).value + pol_obj = common.Policy(policy_name, self.__class__.__name__) + common.pols_binds.store_policy(pol_obj) + commandParseTree = VPNTraffic.convert_pos_expr(commandParseTree, 1) + pol_obj.policy_type = ("classic" + if commandParseTree.upgraded else "advanced") + return [commandParseTree] + + # TODO need to integrate with priority interleaving. + @common.register_for_bind(["User", "Group", "VPN"]) + def convert_vpntraffic_entity_bind(self, commandParseTree, policy_name, + priority_arg, goto_arg): + """ + Handles binding of tmsession policy to AAA user, AAA group + and VPN vserver. + Arguments: + commandParseTree - bind command parse tree. + priority_arg - Indicates whether priority argument is + keyword or positional. It will be either + positional index or keyword name. + goto_arg - Indicates whether gotoPriorityExpression + argument is keyword or positional argument. + It will be either positional index or keyword name. + Returns converted list of parse trees. + """ + policy_type = self.__class__.__name__ + return self.convert_entity_policy_bind( + commandParseTree, commandParseTree, + policy_name, policy_type, priority_arg, goto_arg) + +# TODO VPN class is used to handle VPN global and VPN vserver bindings +# for VPNTraffic policies. Since the VPNTraffic policy conversion is disabled +# for now, disabling the binding conversion as well. Enable this back while enabling +# VPNTraffic policy conversion. +#@common.register_class_methods +class VPN(ConvertConfig): + """ + Handles VPN global and VPN vserver + bind command which can bind the + following policies: + vpn clientlessAccessPolicy, + vpn sessionPolicy, + vpn trafficPolicy + We have to deal with only vpn sessionPolicies + and vpn trafficPolicies. + """ + + # override + flow_type_direction_default = None + + # TODO Create a new class VPNSession, Handle VPN session + # policy conversion in VPNSession class and handle + # VPN session policy bindings to VPN global in + # the below method. + @common.register_for_cmd("bind", "vpn", "global") + def convert_vpn_global(self, commandParseTree): + """ + Handles VPN global bind command. + bind vpn global [-policyName [-priority + ][-gotoPriorityExpression ]] + """ + if not commandParseTree.keyword_exists('policyName'): + return [commandParseTree] + policy_name = commandParseTree.keyword_value("policyName")[0].value + priority_arg = "priority" + goto_arg = "gotoPriorityExpression" + if common.pols_binds.get_policy(policy_name).module == "VPNTraffic": + module = "VPNTraffic" + return self.convert_global_bind(commandParseTree, + commandParseTree, policy_name, + module, priority_arg, goto_arg) + else: + return [commandParseTree] + + @common.register_for_cmd("bind", "vpn", "vserver") + def convert_vpn_vserver_bind(self, commandParseTree): + """ + Handles VPN vserver bind command conversion. + bind vpn vserver [-policy [-priority + ] [-gotoPriorityExpression ] + """ + if not commandParseTree.keyword_exists('policy'): + return [commandParseTree] + + policy_name = commandParseTree.keyword_value("policy")[0].value + policy_type = common.pols_binds.get_policy(policy_name).module + priority_arg = "priority" + goto_arg = "gotoPriorityExpression" + + """ + Calls the method that is registered for the particular + policy type that is bound to vserver. Returns converted_list. + If the policy module is not registered for binding, + then returns the original parse tree. + """ + key = "VPN" + if key in common.bind_table: + if policy_type in common.bind_table[key]: + m = common.bind_table[key][policy_type] + return m.method( + m.obj, commandParseTree, policy_name, priority_arg, + goto_arg) + return [commandParseTree] + + +@common.register_class_methods +class APPFw(ConvertConfig): + """ Handle APPFw feature """ + + @common.register_for_cmd("add", "appfw", "policy") + def convert_policy(self, commandParseTree): + """Convert classic AppFw policy to advanced + Syntax: + add appfw policy + to + add appfw policy + """ + policy_name = commandParseTree.positional_value(0).value + pol_obj = common.Policy(policy_name, self.__class__.__name__) + common.pols_binds.store_policy(pol_obj) + commandParseTree = APPFw.convert_pos_expr(commandParseTree, 1) + pol_obj.policy_type = ("classic" + if commandParseTree.upgraded else "advanced") + return [commandParseTree] + + @common.register_for_cmd("bind", "appfw", "global") + def convert_appfw_global_bind(self, commandParseTree): + """ + Handles appfw global bindinds. + Syntax: + bind appfw global + [] + [-state ( ENABLED | DISABLED )] + """ + + # Remove bind command when state is disabled. + if (commandParseTree.keyword_exists("state") and + commandParseTree.keyword_value("state")[0].value.lower() + == "disabled"): + logging.warning("Following bind command is commented out because" + " state is disabled. If command is required" + " please take a backup because comments will" + " not be saved in ns.conf after triggering" + " 'save ns config': {}" + "".format(str(commandParseTree).strip())) + return ['#' + str(commandParseTree)] + + priority_arg = 1 + goto_arg = 2 + module = self.__class__.__name__ + policy_name = commandParseTree.positional_value(0).value + return self.convert_global_bind(commandParseTree, + commandParseTree, policy_name, module, + priority_arg, goto_arg) + + @common.register_for_bind(["LB"]) + def convert_appfw_vserver_bind(self, commandParseTree, policy_name, + priority_arg, goto_arg): + """ + Handles binding of appfw policy to LB vserver. + Arguments: + commandParseTree - bind command parse tree. + priority_arg - Indicates whether priority argument is + keyword or positional. It will be either + positional index or keyword name. + goto_arg - Indicates whether gotoPriorityExpression + argument is keyword or positional argument. + It will be either positional index or keyword name. + Returns converted list of parse trees. + """ + policy_type = self.__class__.__name__ + return self.convert_entity_policy_bind(commandParseTree, + commandParseTree, policy_name, + policy_type, priority_arg, + goto_arg) + + +@common.register_class_methods +class Syslog(ConvertConfig): + """ Handle Nslog feature """ + + # TODO Classic Syslog policy conversion is disabled for now. The Advanced + # Syslog policies can have Q and S prefixes and SYS.EVAL_CLASSIC_EXPR expression + # which needs to be converted. Registering the Syslog policy to convert_advanced_expr + # in AdvExpression class to support Advance expression conversion. + # While enabling back the Classic Syslog policy conversion, remove the + # entry from convert_advanced_expr. + + # TODO uncomment this when the bind command conversion is supported. + # @common.register_for_cmd("add", "audit", "syslogPolicy") + def convert_policy(self, commandParseTree): + """Convert classic Syslog policy to advanced + Syntax: + add audit syslogPolicy + to + add audit syslogPolicy + """ + policy_name = commandParseTree.positional_value(0).value + pol_obj = common.Policy(policy_name, self.__class__.__name__) + common.pols_binds.store_policy(pol_obj) + commandParseTree = Syslog.convert_pos_expr(commandParseTree, 1) + pol_obj.policy_type = ("classic" + if commandParseTree.upgraded else "advanced") + return [commandParseTree] + + +@common.register_class_methods +class Nslog(ConvertConfig): + """ Handle Nslog feature """ + + # TODO Classic Nslog policy conversion is disabled for now. The Advanced + # Nslog policies can have Q and S prefixes and SYS.EVAL_CLASSIC_EXPR expression + # which needs to be converted. Registering the Nslog policy to convert_advanced_expr + # in AdvExpression class to support Advance expression conversion. + # While enabling back the Classic Nslog policy conversion, remove the + # entry from convert_advanced_expr. + + # TODO uncomment this when the bind command conversion is supported. + # @common.register_for_cmd("add", "audit", "nslogPolicy") + def convert_policy(self, commandParseTree): + """Convert classic Nslog policy to advanced + Syntax: + add audit nslogPolicy + to + add audit nslogPolicy + """ + policy_name = commandParseTree.positional_value(0).value + pol_obj = common.Policy(policy_name, self.__class__.__name__) + common.pols_binds.store_policy(pol_obj) + commandParseTree = Nslog.convert_pos_expr(commandParseTree, 1) + pol_obj.policy_type = ("classic" + if commandParseTree.upgraded else "advanced") + return [commandParseTree] + + +@common.register_class_methods +class Patset(ConvertConfig): + """ Patset entity """ + + @common.register_for_cmd("add", "policy", "patset") + def register_name(self, commandParseTree): + Patset.register_policy_entity_name(commandParseTree) + return [commandParseTree] + + +@common.register_class_methods +class Dataset(ConvertConfig): + """ Dataset entity """ + + @common.register_for_cmd("add", "policy", "dataset") + def register_name(self, commandParseTree): + Dataset.register_policy_entity_name(commandParseTree) + return [commandParseTree] + + +@common.register_class_methods +class HTTP_CALLOUT(ConvertConfig): + """ HTTP callout entity """ + + @common.register_for_cmd("add", "policy", "httpCallout") + def register_name(self, commandParseTree): + callout_name = commandParseTree.positional_value(0).value + lower_callout_name = callout_name.lower() + if (lower_callout_name in classic_entities_names): + """ + This will be true only if the classic named expression has + the same name as the callout entity name. + """ + logging.error(("HTTP callout name {} is conflicting with" + " named expression entity name, please resolve" + " the conflict.").format(callout_name)) + else: + HTTP_CALLOUT.register_policy_entity_name(commandParseTree) + commandParseTree = HTTP_CALLOUT.convert_adv_expr_list( + commandParseTree, ["hostExpr", "urlStemExpr", "headers", + "parameters", "bodyExpr", "fullReqExpr", "resultExpr"]) + return [commandParseTree] + + +@common.register_class_methods +class StringMap(ConvertConfig): + """ String map entity """ + + @common.register_for_cmd("add", "policy", "stringmap") + def register_name(self, commandParseTree): + StringMap.register_policy_entity_name(commandParseTree) + return [commandParseTree] + + +@common.register_class_methods +class NSVariable(ConvertConfig): + """ NS Variable entity """ + + @common.register_for_cmd("add", "ns", "variable") + def register_name(self, commandParseTree): + NSVariable.register_policy_entity_name(commandParseTree) + return [commandParseTree] + + +@common.register_class_methods +class EncryptionKey(ConvertConfig): + """ Encryption key entity """ + + @common.register_for_cmd("add", "ns", "encryptionKey") + def register_name(self, commandParseTree): + EncryptionKey.register_policy_entity_name(commandParseTree) + return [commandParseTree] + + +@common.register_class_methods +class HMACKey(ConvertConfig): + """ HMAC key entity """ + + @common.register_for_cmd("add", "ns", "hmacKey") + def register_name(self, commandParseTree): + HMACKey.register_policy_entity_name(commandParseTree) + return [commandParseTree] + + +@common.register_class_methods +class NamedExpression(ConvertConfig): + """ Handle Named expression feature """ + + # Built-in classic named expression names and there + # corresponding built-in advanced named expression names. + built_in_named_expr = { + "ns_true": "TRUE", + "ns_false": "FALSE", + "ns_non_get": "ns_non_get_adv", + "ns_cachecontrol_nostore": "ns_cachecontrol_nostore_adv", + "ns_cachecontrol_nocache": "ns_cachecontrol_nocache_adv", + "ns_header_pragma": "ns_header_pragma_adv", + "ns_header_cookie": "ns_header_cookie_adv", + "ns_ext_cgi": "ns_ext_cgi_adv", + "ns_ext_asp": "ns_ext_asp_adv", + "ns_ext_exe": "ns_ext_exe_adv", + "ns_ext_cfm": "ns_ext_cfm_adv", + "ns_ext_ex": "ns_ext_ex_adv", + "ns_ext_shtml": "ns_ext_shtml_adv", + "ns_ext_htx": "ns_ext_htx_adv", + "ns_url_path_cgibin": "ns_url_path_cgibin_adv", + "ns_url_path_exec": "ns_url_path_exec_adv", + "ns_url_path_bin": "ns_url_path_bin_adv", + "ns_url_tokens": "ns_url_tokens_adv", + "ns_ext_not_gif": "ns_ext_not_gif_adv", + "ns_ext_not_jpeg": "ns_ext_not_jpeg_adv", + "ns_cmpclient": "ns_cmpclient_adv", + "ns_slowclient": "ns_slowclient_adv", + "ns_content_type": "ns_content_type_advanced", + "ns_msword": "ns_msword_advanced", + "ns_msexcel": "ns_msexcel_advanced", + "ns_msppt": "ns_msppt_advanced", + "ns_css": "ns_css_adv", + "ns_xmldata": "ns_xmldata_adv", + "ns_mozilla_47": "ns_mozilla_47_adv", + "ns_msie": "ns_msie_adv" + } + + @staticmethod + def register_built_in_named_exprs(): + """ + Register built-in classic Named expression names in + classic_entities_names and built-in advanced Named expression names + in policy_entities_names. + """ + for classic_exp_name in NamedExpression.built_in_named_expr: + classic_entities_names.add(classic_exp_name) + policy_entities_names.add(NamedExpression.built_in_named_expr[ + classic_exp_name].lower()) + + @common.register_for_cmd("add", "policy", "expression") + def convert_policy(self, commandParseTree): + """ + Classic named expression name is not + valid for advanced expression if: + 1. It the name is same as one of the Policy + entity (patset/dataset/stringmap/ + variable/hmacKey/EncriptionKey/callout) name. + 2. it doesn't start with ASCII alphabetic character or underscore. + 3. it has characters other than ASCII alphanumerics + or underscore characters. + 4. it is equal to a advanced policy expression reserved word (prefix identifier or + enum value) + """ + reserved_word_list = set( + [ # Advanced policy expression prefix list + "subscriber", + "connection", + "analytics", + "diameter", + "target", + "server", + "radius", + "oracle", + "extend", + "client", + "mysql", + "mssql", + "false", + "true", + "text", + "smpp", + "icap", + "http", + "url", + "sys", + "sip", + "ica", + "dns", + "aaa", + "s", + "q", + "re", + "xp", + "ce" + ]) + + expr_name = commandParseTree.positional_value(0).value + expr_rule = commandParseTree.positional_value(1).value + named_expr[expr_name] = expr_rule + lower_expr_name = expr_name.lower() + if (((lower_expr_name in reserved_word_list) or + (re.match('^[a-z_][a-z0-9_]*$', lower_expr_name) is None) or + (lower_expr_name in policy_entities_names))): + logging.error(("Expression name {} is invalid for advanced " + "expression: names must begin with an ASCII " + "alphabetic character or underscore and must " + "contain only ASCII alphanumerics or underscores" + " and shouldn't be name of another policy entity" + "; words reserved for policy use may not be used;" + " underscores will be substituted for any invalid" + " characters in corresponding advanced name") + .format(expr_name)) + + if commandParseTree.keyword_exists('clientSecurityMessage'): + logging.error(("Error in converting expression {} : " + "conversion of clientSecurityMessage based " + "expression is not supported.") + .format(expr_name)) + return [commandParseTree] + + original_tree = copy.deepcopy(commandParseTree) + """Convert classic named expression to advanced + Syntax: + add policy expression + to + add policy expression + """ + commandParseTree = NamedExpression \ + .convert_pos_expr(commandParseTree, 1) + + if commandParseTree.adv_upgraded: + tree_list = [commandParseTree] + else: + tree_list = [original_tree] + if commandParseTree.upgraded: + """ + Because we are not currently converting all the commands that + use named expressions, we can have the situation where a + non-converted command uses a Classic named expression but a + converted command would have used the same named expression. + To deal with this we create an Advanced that is equivalent + to the old Classic, give it a new name, and replace all the + references to the old Classic in converted expressions to the + corresponding Advanced. Because of that we need to return + both the old Classic and corresponding Advanced named + expressions from this routine. + """ + name_node = commandParseTree.positional_value(0) + name_node.set_value(get_advanced_name(name_node.value)) + tree_list.append(commandParseTree) + NamedExpression.register_policy_entity_name(commandParseTree) + NamedExpression.register_classic_entity_name(original_tree) + else: + NamedExpression.register_policy_entity_name(original_tree) + return tree_list + + +@common.register_class_methods +class HTTPProfile(ConvertConfig): + """ Handle HTTP Profile """ + + @common.register_for_cmd("add", "ns", "httpProfile") + @common.register_for_cmd("set", "ns", "httpProfile") + def convert_spdy(self, commandParseTree): + """Convert spdy feature to HTTP2 + Syntax: + add ns httpProfile -spdy + to + add ns httpProfile -http2 ENABLED + """ + if commandParseTree.keyword_exists('spdy'): + commandParseTree.remove_keyword('spdy') + http2_keyword = CLIKeywordParameter(CLIKeywordName("http2")) + http2_keyword.add_value('ENABLED') + commandParseTree.add_keyword(http2_keyword) + commandParseTree = HTTPProfile \ + .convert_adv_expr_list(commandParseTree, ["clientIpHdrExpr"]) + return [commandParseTree] + + +@common.register_class_methods +class ContentSwitching(ConvertConfig): + """ Handle Content Switching feature """ + + # override + bind_default_goto = None + + def __init__(self): + """ + _policy_bind_info - Contains information about classic policies + without actions and there bind commands. + key - policy name + value - dictionary with the following keys: + "policy_tree" - Policy tree without action + "bind_trees" - List of bind trees where + the corresponding policy is + bound. + """ + self._policy_bind_info = OrderedDict() + + @common.register_for_cmd("add", "cs", "vserver") + def convert_cs_vserver(self, commandParseTree): + """ + Get vserver protocol to help in filter bind conversion + cs_protocol - cs vserver protocol + csv_name - cs vserver name + vserver_protocol_dict - dict to store protocol as value to the + vserver name as key + """ + cs_protocol = commandParseTree.positional_value(1).value + csv_name = commandParseTree.positional_value(0).value + vserver_protocol_dict[csv_name] = cs_protocol.upper() + commandParseTree = ContentSwitching.convert_adv_expr_list( + commandParseTree, ["Listenpolicy", "pushLabel"]) + return [commandParseTree] + + @common.register_for_cmd("add", "cs", "policy") + def convert_policy(self, commandParseTree): + """Convert CS policy to advanced + Syntax: + Conversion happens as: + 1. add cs policy -domain + to + add cs policy -rule HTTP.REQ.HOSTNAME.EQ("") + + 2. add cs policy -rule + to + add cs policy -rule + + 3. add cs policy -rule -domain + to + add cs policy -rule + ' && HTTP.REQ.HOSTNAME.EQ("")' + + 4. add cs policy -url + to + add cs policy -rule + + 5. add cs policy -url -domain + to + add cs policy -rule ")> + + """ + policy_name = commandParseTree.positional_value(0).value + pol_obj = common.Policy(policy_name, self.__class__.__name__) + common.pols_binds.store_policy(pol_obj) + """ Only in advanced policy, action can be present. + """ + if commandParseTree.keyword_exists('action'): + pol_obj.policy_type = "advanced" + commandParseTree = ContentSwitching.convert_adv_expr_list( + commandParseTree, ["rule"]) + return [commandParseTree] + if commandParseTree.keyword_exists('rule'): + if commandParseTree.keyword_exists('domain'): + rule_node = commandParseTree.keyword_value('rule') + rule_expr = rule_node[0].value + converted_expr = convert_classic_expr.convert_classic_expr( + rule_expr) + if converted_expr is None: + logging.error('Error in converting command : ' + + str(commandParseTree)) + return [commandParseTree] + converted_expr = converted_expr.strip('"') + domain_name = commandParseTree.keyword_value('domain')[0] \ + .value + domain_rule = 'HTTP.REQ.HOSTNAME.EQ(\\"' + \ + domain_name + '\\")' + commandParseTree.remove_keyword('domain') + complete_expr = '"(' + converted_expr + ') && ' + \ + domain_rule + '"' + rule_node[0].set_value(complete_expr, True) + commandParseTree.set_upgraded() + else: + commandParseTree = ContentSwitching \ + .convert_keyword_expr(commandParseTree, 'rule') + elif commandParseTree.keyword_exists('url'): + domain_rule = None + converted_url_expr = None + prefix = None + prefix_val = None + suffix = None + url_expr = commandParseTree.keyword_value('url')[0].value + if url_expr.endswith('.'): + converted_url_expr = 'HTTP.REQ.URL.PATH.EQ("' + \ + url_expr + '.")' + else: + prefix_suffix = url_expr.rsplit('.', 1) + if len(prefix_suffix) is 1: + """ No suffix is present in URL.""" + prefix = prefix_suffix[0] + suffix = None + if prefix.endswith('*'): + prefix_val = prefix[:-1] + else: + """ Suffix is present in URL.""" + prefix = prefix_suffix[0] + suffix = prefix_suffix[1] + """ + If URL is abc..*.html, then + in classic code, we don't check + one dot before *, and this happens + only if there is some suffix. + """ + if prefix.endswith('*'): + prefix_val = prefix[:-1] + if prefix_val.endswith('.'): + prefix_val = prefix_val[:-1] + + if suffix and (prefix != '/') and (not prefix.endswith('*')): + converted_url_expr = 'HTTP.REQ.URL.PATH.EQ("' + \ + url_expr + '")' + elif (suffix is None) and (not prefix.endswith('*')): + converted_url_expr = 'HTTP.REQ.URL.PATH.EQ(("' + \ + prefix + '." + HTTP.REQ.URL.SUFFIX).' + \ + 'STRIP_END_CHARS("."))' + elif (prefix == '/*') and (suffix is not None): + converted_url_expr = 'HTTP.REQ.URL.SUFFIX.EQ("' + \ + suffix + '")' + elif (prefix.endswith('*')) and (suffix is not None): + converted_url_expr = '(HTTP.REQ.URL.STARTSWITH("' + \ + prefix_val + '") && HTTP.REQ.URL.SUFFIX.EQ("' + \ + suffix + '"))' + elif (prefix == '/*'): + converted_url_expr = 'true' + elif (prefix.endswith('*')): + converted_url_expr = 'HTTP.REQ.URL.STARTSWITH("' + \ + prefix_val + '")' + elif (suffix is not None) and (prefix == '/'): + converted_url_expr = 'HTTP.REQ.URL.SUFFIX.EQ("' + \ + suffix + '")' + + if commandParseTree.keyword_exists('domain'): + domain_name = commandParseTree.keyword_value('domain')[0] \ + .value + domain_rule = 'HTTP.REQ.HOSTNAME.EQ("' + domain_name + '")' + commandParseTree.remove_keyword('domain') + + if (domain_rule): + converted_url_expr = converted_url_expr + ' && ' + domain_rule + + commandParseTree.remove_keyword('url') + rule_keyword = CLIKeywordParameter(CLIKeywordName('rule')) + rule_keyword.add_value(converted_url_expr) + commandParseTree.add_keyword(rule_keyword) + elif commandParseTree.keyword_exists('domain'): + domain_name = commandParseTree.keyword_value('domain')[0].value + domain_rule = 'HTTP.REQ.HOSTNAME.EQ("' + domain_name + '")' + commandParseTree.remove_keyword('domain') + rule_keyword = CLIKeywordParameter(CLIKeywordName("rule")) + rule_keyword.add_value(domain_rule) + commandParseTree.add_keyword(rule_keyword) + + if commandParseTree.upgraded: + pol_obj.policy_type = "classic" + # Saving policies for resolving multiple bind points for + # CS policies without action issue. + self._policy_bind_info[policy_name] = {} + self._policy_bind_info[policy_name]["policy_tree"] = \ + commandParseTree + return [] + else: + pol_obj.policy_type = "advanced" + return [commandParseTree] + + @common.register_for_cmd("bind", "cs", "vserver") + def convert_cs_bind_command(self, commandParseTree): + """ + Handles CS vserver bind command. + bind cs vserver -policyName + -priority [-gotoPriorityExpression ] + """ + if not commandParseTree.keyword_exists('policyName'): + return [commandParseTree] + + # Get the policy name + policy_name = commandParseTree.keyword_value('policyName')[0].value + priority_arg = "priority" + goto_arg = "gotoPriorityExpression" + + class_name = self.__class__.__name__ + policy_type = common.pols_binds.get_policy(policy_name).module + if policy_type == class_name: + if policy_name in self._policy_bind_info: + # Saving bind commands for resolving multiple bind points for + # CS policies without action issue. + if "bind_trees" not in self._policy_bind_info[policy_name]: + self._policy_bind_info[policy_name]["bind_trees"] = [] + self._policy_bind_info[policy_name]["bind_trees"].append( + commandParseTree) + return [] + else: + return self.convert_entity_policy_bind( + commandParseTree, commandParseTree, + policy_name, policy_type, priority_arg, goto_arg) + + """ + Calls the method that is registered for the particular + policy type that is bound to CS. Returns converted_list. + If the policy module is not registered for binding, + then returns the original parse tree. + """ + key = "ContentSwitching" + if key in common.bind_table: + if policy_type in common.bind_table[key]: + m = common.bind_table[key][policy_type] + return m.method(m.obj, commandParseTree, policy_name, + priority_arg, goto_arg) + return [commandParseTree] + + @common.register_for_bind(["CacheRedirection"]) + def convert_cs_policy_entity_bind( + self, commandParseTree, policy_name, priority_arg, goto_arg): + """ + Handles CS policy binding to CR vserver. + Arguments: + commandParseTree - bind command parse tree. + priority_arg - Indicates whether priority argument is + keyword or positional. It will be either + positional index or keyword name. + goto_arg - Indicates whether gotoPriorityExpression + argument is keyword or positional argument. + It will be either positional index or keyword name. + Returns converted list of parse trees. + """ + policy_type = self.__class__.__name__ + if policy_name in self._policy_bind_info: + # Saving bind commands for resolving multiple bind points for + # CS policies without action issue. + if "bind_trees" not in self._policy_bind_info[policy_name]: + self._policy_bind_info[policy_name]["bind_trees"] = [] + self._policy_bind_info[policy_name]["bind_trees"].append( + commandParseTree) + return [] + else: + return self.convert_entity_policy_bind( + commandParseTree, commandParseTree, + policy_name, policy_type, priority_arg, goto_arg) + + @common.register_for_final_call + def get_converted_cmds(self): + """ + Classic CS policies do not support CS action. But in advanced policies + there is limitation that multiple bind points are not allowed for + CS policies without action. This will lead to issue during the + conversion if any classic CS policy is bound to multiple bind points. + To avoid this issue following steps are followed for each bind command: + 1. Get policy name and vserver name from bind command. + 2. New action is created with name "nspepi_adv_cs_act_" + 3. New policy is created with name + "nspepi_adv__" and + set action keyword to "nspepi_adv_cs_act_" + 4. Remove targetLBVserver from bind command and update the policy name + to newly created policy name. + 5. If same policy is bound to different vservers, multiple policies + are created. + CS policies can be bound to CS vserver and CR vserver. + Return list of newly added CS actions and policies. + """ + newly_added_policy_names = [] + newly_added_action_names = [] + pol_list = [] + act_list = [] + overlength_action_names = {} + overlength_policy_names = {} + overlength_action_counter = 0 + overlength_policy_counter = 0 + for policy_name in self._policy_bind_info: + policy_tree = self._policy_bind_info[policy_name]["policy_tree"] + if "bind_trees" not in self._policy_bind_info[policy_name]: + # when policy is not used in any bind command. + pol_list.append(policy_tree) + continue + for index in range(len(self._policy_bind_info[policy_name][ + "bind_trees"])): + vserver_name = "" + bind_tree = self._policy_bind_info[policy_name][ + "bind_trees"][index] + if ((' '.join(bind_tree.get_command_type())).lower() == + "bind cs vserver"): + vserver_name = bind_tree.keyword_value( + "targetLBVserver")[0].value + elif ((' '.join(bind_tree.get_command_type())).lower() == + "bind cr vserver"): + vserver_name = bind_tree.keyword_value( + "policyName")[1].value + new_policy_name = "nspepi_adv_" + policy_name + '_' + \ + vserver_name + truncated_pol_name = new_policy_name + action_name = "nspepi_adv_cs_act_" + vserver_name + truncated_act_name = action_name + if new_policy_name not in newly_added_policy_names: + if action_name not in newly_added_action_names: + # Create new action + action_tree = CLICommand("add", "cs", "action") + # Check action name length. Max allowed length is 127 + if len(action_name) > 127: + truncated_act_name, overlength_action_counter = \ + self.truncate_name(action_name, + overlength_action_names, + overlength_action_counter) + pos = CLIPositionalParameter(truncated_act_name) + action_tree.add_positional(pos) + vserver_key = CLIKeywordParameter(CLIKeywordName( + "targetLBVserver")) + vserver_key.add_value(vserver_name) + action_tree.add_keyword(vserver_key) + act_list.append(action_tree) + newly_added_action_names.append(action_name) + else: + # when action is already added. + # Get truncated name if truncated. + if action_name in overlength_action_names: + truncated_act_name = overlength_action_names[ + action_name] + # Create new policy with [policy_name]_[vserver_name] as + # as name and bind to newly created action + # cs_act_[vserver_name] + new_policy = copy.deepcopy(policy_tree) + # Max length of policy name allowed is 127. + truncated_pol_name = new_policy_name + if len(new_policy_name) > 127: + truncated_pol_name, overlength_policy_counter = \ + self.truncate_name(new_policy_name, + overlength_policy_names, + overlength_policy_counter) + self.update_tree_arg(new_policy, 0, truncated_pol_name) + action_key = CLIKeywordParameter(CLIKeywordName("action")) + action_key.add_value(truncated_act_name) + new_policy.add_keyword(action_key) + newly_added_policy_names.append(new_policy_name) + pol_list.append(new_policy) + else: + # When policy is already added. + # Get truncated policy name if truncated. + if new_policy_name in overlength_policy_names: + truncated_pol_name = overlength_policy_names[ + new_policy_name] + # Remove targetLBVserver from bind command and update policy + # name to newly added policy name. + priority_arg = "priority" + goto_arg = "gotoPriorityExpression" + policy_type = self.__class__.__name__ + self.update_tree_arg(bind_tree, "policyName", + truncated_pol_name) + if ((' '.join(bind_tree.get_command_type())).lower() == + "bind cs vserver"): + bind_tree.remove_keyword("targetLBVserver") + elif ((' '.join(bind_tree.get_command_type())).lower() == + "bind cr vserver"): + # In bind cr vserver command, vserver name exists in + # following way: + # bind cr vserver -policyName + # + bind_tree.remove_keyword_value("policyName", 1) + self.convert_entity_policy_bind( + bind_tree, bind_tree, policy_name, + policy_type, priority_arg, goto_arg) + return act_list + pol_list + + def truncate_name(self, name, name_mapping, counter): + """ + Truncates name shorter than 127 and adds a counter at the end. + name - name that should be truncated. + name_mapping - dictionary which saves name and its truncated name. + key - name + value - truncated name + counter - counter to be appended at the end of truncated name. + """ + counter += 1 + # Reserving 1 for '_' + 6 for counter. + truncated_name = name[0: 120] + truncated_name += "_" + str(counter) + name_mapping[name] = truncated_name + return truncated_name, counter + + +@common.register_class_methods +class AAA(ConvertConfig): + + @common.register_for_cmd("add", "aaa", "group") + def convert_add_group(self, tree): + """ + Process: add aaa group [-weight ] + and store weight for each group. + + Args: + tree: Command parse tree for add aaa group command + + Returns: + tree: Processed command parse tree for add aaa group command + """ + groupname = common.get_cmd_arg(0, tree) + weight = common.get_cmd_arg("weight", tree) + weight = weight if weight else "0" + common.pols_binds.store_group(common.Group(groupname, weight)) + return [tree] + + def user_group_bind_common(self, tree, key): + """ + Common processing for bind aaa user and bind aaa group. + bind aaa user [-policy ] + [-priority ] [-type ] + [-gotoPriorityExpression ] ... + bind aaa group [-policy ] + [-priority ] [-type ] + [-gotoPriorityExpression ] ... + """ + policy_name = common.get_cmd_arg("policy", tree) + if not policy_name: + return [tree] + policy_type = common.pols_binds.get_policy(policy_name).module + priority_arg = "priority" + goto_arg = "gotoPriorityExpression" + + # Calls the method that is registered for the particular + # policy type that is bound to the user or group. + # Returns converted_list. If the policy module is not + # registered for binding, then returns the original parse + # tree. + if key in common.bind_table: + if policy_type in common.bind_table[key]: + m = common.bind_table[key][policy_type] + return m.method(m.obj, tree, policy_name, priority_arg, + goto_arg) + return [tree] + + @common.register_for_cmd("bind", "aaa", "user") + def convert_user_bind(self, tree): + return self.user_group_bind_common(tree, "User") + + @common.register_for_cmd("bind", "aaa", "group") + def convert_group_bind(self, tree): + return self.user_group_bind_common(tree, "Group") + +@common.register_class_methods +class AdvExpression(ConvertConfig): + """ + Handles conversion of Q and S prefixes and SYS.EVAL_CLASSIC_EXPR expression in commands + which allows only advanced expressions. + """ + + @common.register_for_cmd("add", "videooptimization", "detectionpolicy") + @common.register_for_cmd("add", "videooptimization", "pacingpolicy") + @common.register_for_cmd("add", "dns", "policy") + @common.register_for_cmd("add", "cache", "selector") + @common.register_for_cmd("add", "cs", "action") + @common.register_for_cmd("add", "vpn", "clientlessAccessPolicy") + @common.register_for_cmd("add", "authentication", "webAuthAction") + @common.register_for_cmd("set", "authentication", "webAuthAction") + @common.register_for_cmd("add", "tm", "trafficPolicy") + @common.register_for_cmd("add", "authentication", "samlIdPPolicy") + @common.register_for_cmd("add", "feo", "policy") + @common.register_for_cmd("add", "cache", "policy") + @common.register_for_cmd("add", "transform", "policy") + @common.register_for_cmd("add", "appqoe", "action") + @common.register_for_cmd("add", "appqoe", "policy") + @common.register_for_cmd("add", "ssl", "policy") + @common.register_for_cmd("add", "appflow", "policy") + @common.register_for_cmd("add", "autoscale", "policy") + @common.register_for_cmd("add", "authentication", "Policy") + @common.register_for_cmd("add", "authentication", "loginSchemaPolicy") + @common.register_for_cmd("add", "authentication", "loginSchema") + @common.register_for_cmd("add", "gslb", "vserver") + @common.register_for_cmd("add", "ns", "assignment") + @common.register_for_cmd("add", "dns", "action64") + @common.register_for_cmd("add", "dns", "policy64") + @common.register_for_cmd("add", "authentication", "OAuthIdPPolicy") + @common.register_for_cmd("add", "authentication", "samlIdPProfile") + @common.register_for_cmd("add", "contentInspection", "policy") + @common.register_for_cmd("add", "ica", "policy") + @common.register_for_cmd("add", "lb", "group") + @common.register_for_cmd("add", "audit", "messageaction") + @common.register_for_cmd("add", "aaa", "preauthenticationpolicy") + @common.register_for_cmd("add", "spillover", "policy") + @common.register_for_cmd("add", "stream", "selector") + @common.register_for_cmd("add","tm", "formSSOAction") + @common.register_for_cmd("add", "tm", "samlSSOProfile") + @common.register_for_cmd("add", "vpn", "sessionPolicy") + @common.register_for_cmd("add", "vpn", "trafficAction") + @common.register_for_cmd("add", "vpn", "vserver") + # TODO: This entry need to be removed when Classic Syslog policy + # conversion is enabled in Syslog class. + @common.register_for_cmd("add", "audit", "syslogPolicy") + # TODO: This entry need to be removed when Classic Nslog policy + # conversion is enabled in Nslog class. + @common.register_for_cmd("add", "audit", "nslogPolicy") + # TODO: This entry need to be removed when Classic authorization policy + # conversion is enabled in Authorization class. + @common.register_for_cmd("add", "authorization", "policy") + # TODO: This entry need to be removed when Classic VPNTraffic policy + # conversion is enabled in VPNTraffic class. + @common.register_for_cmd("add", "vpn", "trafficPolicy") + def convert_advanced_expr(self, tree): + """ + Commands which allows ONLY advanced expressions should be registered for this method. + Handles conversion of Q and S prefixes and SYS.EVAL_CLASSIC_EXPR expression. + Each command that will be registered to this method, should add an entry in + command_parameters_list. + """ + + # Each command should mention the list of parameters where advanced expression + # can be used. Only these parameters will be checked for SYS.EVAL_CLASSIC_EXPR + # expression. + # If its a keyword parameter, mention the keyword name. + # If its a positional parameter, mention the position of the parameter. + command_parameters_list = { + "add videooptimization detectionpolicy": ["rule"], + "add videooptimization pacingpolicy": ["rule"], + "add dns policy": [1], + "add cache selector": [1, 2, 3, 4, 5, 6, 7, 8], + "add cs action": ["targetVserverExpr"], + "add vpn clientlessaccesspolicy": [1], + "add authentication webauthaction": ["fullReqExpr", "successRule"], + "set authentication webauthaction": ["fullReqExpr", "successRule"], + "add tm trafficpolicy": [1], + "add authentication samlidppolicy": ["rule"], + "add feo policy": [1], + "add cache policy": ["rule"], + "add transform policy": [1], + "add appqoe action": ["dosTrigExpression"], + "add appqoe policy": ["rule"], + "add ssl policy": ["rule"], + "add appflow policy": [1], + "add autoscale policy": ["rule"], + "add authentication policy": ["rule"], + "add authentication loginschemapolicy": ["rule"], + "add authentication loginschema": ["userExpression", "passwdExpression"], + "add gslb vserver": ["rule"], + "add ns assignment": ["set", "append", "add", "sub"], + "add dns action64": ["mappedRule", "excludeRule"], + "add dns policy64": ["rule"], + "add authentication oauthidppolicy": ["rule"], + "add authentication samlidpprofile": ["NameIDExpr", "acsUrlRule"], + "add contentinspection policy": ["rule"], + "add ica policy": ["rule"], + "add lb group": ["rule"], + "add audit messageaction": [2], + "add aaa preauthenticationpolicy": [1], + "add spillover policy": ["rule"], + "add stream selector": [1, 2, 3, 4, 5], + "add tm formssoaction": ["ssoSuccessRule"], + "add tm samlssoprofile": ["relaystateRule", "NameIDExpr"], + "add vpn sessionpolicy": [1], + "add vpn trafficaction": ["userExpression", "passwdExpression"], + "add vpn vserver": ["Listenpolicy"], + # TODO: This entry need to be removed when Classic Syslog policy + # conversion is enabled in Syslog class. + "add audit syslogpolicy": [1], + # TODO: This entry need to be removed when Classic Nslog policy + # conversion is enabled in Nslog class. + "add audit nslogpolicy": [1], + # TODO: This entry need to be removed when Classic authorization policy + # conversion is enabled in Authorization class. + "add authorization policy": [1], + # TODO: This entry need to be removed when Classic VPNTraffic policy + # conversion is enabled in VPNTraffic class. + "add vpn trafficpolicy": [1], + } + + command = " ".join(tree.get_command_type()).lower() + if command in command_parameters_list: + tree = AdvExpression.convert_adv_expr_list(tree, command_parameters_list[command]) + return [tree] diff --git a/nspepi/nspepi2/convert_cmp_cmd.py b/nspepi/nspepi2/convert_cmp_cmd.py new file mode 100644 index 0000000..3aaac56 --- /dev/null +++ b/nspepi/nspepi2/convert_cmp_cmd.py @@ -0,0 +1,540 @@ +#!/usr/bin/env python2 + +# Copyright 2021 Citrix Systems, Inc. All rights reserved. +# Use of this software is governed by the license terms, if any, +# which accompany or are included with this software. + +import logging +from collections import OrderedDict + +import nspepi_common as common +import nspepi_parse_tree +import convert_cli_commands as cli_cmds + + +@common.register_class_methods +class CMP(cli_cmds.ConvertConfig): + """ + Converts classic CMP policies and + bind command. + CMP policies can be bound to cmp global, + LB, CS, CR vserver. + """ + + # override + flow_type_direction_default = "RESPONSE" + # Classic built-in policy names and there corresponding + # advanced built-in policy names. + built_in_policies = { + "ns_cmp_content_type": "ns_adv_cmp_content_type", + "ns_cmp_msapp": "ns_adv_cmp_msapp", + "ns_cmp_mscss": "ns_adv_cmp_mscss", + "ns_nocmp_mozilla_47": "ns_adv_nocmp_mozilla_47", + "ns_nocmp_xml_ie": "ns_adv_nocmp_xml_ie" + } + + def __init__(self): + """ + Information about CMP commands. + _cmp_bind_info - Contains CMP policies binding info. + key - bind point(possible values: + ""(which indicates the compression global + bind point) or ). + value - dictionary with the following keys + "bind_parse_trees", + "is_classic_policy_bound", + "is_advanced_policy_bound". + _initial_cmp_parameter- Initial CMP parameter policy type. + _classic_builtin_bind - Contains info about classic builtin policy + bindings. + key - bind command parse tree in which classic + built-in policy name is replaced with the + corresponding advanced built-in policy + name. + value - classic built-in policy name. + """ + self._cmp_bind_info = OrderedDict() + self._initial_cmp_parameter = "advanced" + self._classic_builtin_bind = OrderedDict() + + @common.register_for_init_call + def store_builtin_cmp_policies(self): + """ + Creates and stores Policy object for built-in CMP policies. + """ + self.store_builtin_policies() + + @common.register_for_cmd("set", "cmp", "parameter") + def set_cmp_parameter(self, cmp_param_tree): + """ + Remove policyType parameter from the command + Syntax: + set cmp parameter -policyType ADVANCED + """ + if cmp_param_tree.keyword_exists("policyType"): + self._initial_cmp_parameter = \ + cmp_param_tree.keyword_value("policyType")[0].value.lower() + cmp_param_tree.remove_keyword("policyType") + if cmp_param_tree.get_number_of_params() == 0: + return [] + return [cmp_param_tree] + + @common.register_for_cmd("set", "cmp", "policy") + def set_cmp_policy(self, cmp_policy_tree): + """ + Classic CMP built-in policies cannot be changed using set command. + Advanced CMP built-in policies can be changed using set command and + set command is saved in ns.conf. Since we are replacing the classic + built-in policy name with corresponding advanced built-in policy name, + if advanced policy is changed with set command, then functionality + will change. So, if advanced built-in policy is changed then add new + advanced policy which will be equivalent to classic built-in policy. + cmp_policy_tree - set cmp policy command parse tree + """ + advanced_builtin_policy = { + "ns_adv_cmp_content_type": { + "classic": "ns_cmp_content_type", + "rule": "HTTP.RES.HEADER(\"Content-Type\")" + + ".CONTAINS(\"text\")", + "resAction": "COMPRESS" + }, + "ns_adv_cmp_msapp": { + "classic": "ns_cmp_msapp", + "rule": "ns_msie_adv && (HTTP.RES.HEADER" + + "(\"Content-Type\").CONTAINS(\"appl" + + "ication/msword\") || HTTP.RES.HEADE" + + "R(\"Content-Type\").CONTAINS(\"" + + "application/vnd.ms-excel\") || " + + "HTTP.RES.HEADER(\"Content-Type\")" + + ".CONTAINS(\"application/vnd.ms-" + + "powerpoint\"))", + "resAction": "COMPRESS" + }, + "ns_adv_cmp_mscss": { + "classic": "ns_cmp_mscss", + "rule": "ns_msie_adv && HTTP.RES.HEADER" + + "(\"Content-Type\").CONTAINS" + + "(\"text/css\")", + "resAction": "COMPRESS" + }, + "ns_adv_nocmp_mozilla_47": { + "classic": "ns_nocmp_mozilla_47", + "rule": "HTTP.REQ.HEADER(\"User-Agent\")." + + "CONTAINS(\"Mozilla/4.7\") && HTTP." + + "RES.HEADER(\"Content-Type\")." + + "CONTAINS(\"text/css\")", + "resAction": "NOCOMPRESS" + }, + "ns_adv_nocmp_xml_ie": { + "classic": "ns_nocmp_xml_ie", + "rule": "ns_msie_adv && HTTP.RES.HEADER" + + "(\"Content-Type\").CONTAINS" + + "(\"text/xml\")", + "resAction": "NOCOMPRESS" + } + } + tree_list = [cmp_policy_tree] + adv_policy_name = cmp_policy_tree.positional_value(0).value + if adv_policy_name in advanced_builtin_policy: + # Tree construction + policy_name = "nspepi_adv_" + adv_policy_name + policy_tree = nspepi_parse_tree.CLICommand("add", "cmp", "policy") + pos = nspepi_parse_tree.CLIPositionalParameter(policy_name) + policy_tree.add_positional(pos) + rule_key = nspepi_parse_tree.CLIKeywordParameter( + nspepi_parse_tree.CLIKeywordName("rule")) + rule_key.add_value( + advanced_builtin_policy[adv_policy_name]["rule"]) + policy_tree.add_keyword(rule_key) + action_key = nspepi_parse_tree.CLIKeywordParameter( + nspepi_parse_tree.CLIKeywordName("resAction")) + action_key.add_value( + advanced_builtin_policy[adv_policy_name]["resAction"]) + policy_tree.add_keyword(action_key) + tree_list.append(policy_tree) + # Update policy name in built_in_policies dictionary. + self.built_in_policies[advanced_builtin_policy[adv_policy_name][ + "classic"]] = policy_name + return tree_list + + @common.register_for_cmd("add", "cmp", "policy") + def convert_cmp_policy(self, cmp_policy_tree): + """ + Converts classic cmp policy to advanced. + Syntax: + add cmp policy -rule + -resAction + Converts to + add cmp policy -rule + -resAction + """ + policy_name = cmp_policy_tree.positional_value(0).value + pol_obj = common.Policy(policy_name, self.__class__.__name__) + common.pols_binds.store_policy(pol_obj) + cli_cmds.ConvertConfig.convert_keyword_expr(cmp_policy_tree, 'rule') + pol_obj.policy_type = ("classic" + if cmp_policy_tree.upgraded else "advanced") + return [cmp_policy_tree] + + @common.register_for_cmd("bind", "cmp", "global") + def convert_cmp_global_bind(self, bind_cmd_tree): + """ + Handles CMP policy bindings to cmp global. + Syntax for classic policy binding: + bind cmp global [-priority ] + [-state (ENABLED/DISABLED)] + When classic CMP policy is bound: + 1. If -state is DISABLED, comment the bind command. + 2. Add -type RES_DEFAULT keyword. + 3. Throw error when functionality may change. + """ + # Comment the bind command and throw warning when + # state is disabled. + if bind_cmd_tree.keyword_exists("state") and \ + bind_cmd_tree.keyword_value("state")[0].value.lower() == \ + "disabled": + logging.warning(( + "Following bind command is commented out because" + " state is disabled. If command is required please take" + " a backup because comments will not be saved in ns.conf" + " after triggering 'save ns config': {}"). + format(str(bind_cmd_tree).strip()) + ) + return ["#" + str(bind_cmd_tree)] + + policy_name = bind_cmd_tree.positional_value(0).value + self.replace_builtin_policy(bind_cmd_tree, policy_name, 0) + bind_point = "" + self.update_bind_info(bind_cmd_tree, bind_point) + return [] + + @common.register_for_bind(["LB", "ContentSwitching", "CacheRedirection"]) + def convert_cmp_policy_vserver_bind( + self, bind_cmd_tree, policy_name, priority_arg, goto_arg): + """ + Handles CMP policy binding to LB vserver, + CS vserver, CR vserver. + Syntax for classic CMP policy binding: + bind lb/cr/cs vserver -policyName + [-priority ] + when classic cmp policy is bound: + 1. Add -type RESPONSE keyword. + 2. Throw error when functionality may change. + """ + vserver_name = bind_cmd_tree.positional_value(0).value + self.replace_builtin_policy(bind_cmd_tree, policy_name, "policyName") + bind_point = vserver_name + self.update_bind_info(bind_cmd_tree, bind_point) + return [] + + def replace_builtin_policy(self, bind_cmd_tree, policy_name, policy_arg): + """ + If bound policy is classic built-in policy, then replace policy name + with advanced built-in policy. + """ + if policy_name in self.built_in_policies: + # Update policy name to advanced policy name. + self.update_tree_arg(bind_cmd_tree, policy_arg, + self.built_in_policies[policy_name]) + self._classic_builtin_bind[bind_cmd_tree] = policy_name + + def update_bind_info(self, bind_cmd_tree, bind_point): + """ + Appends bind command parse tree to _cmp_bind_info + and updates the bind_info. + bind_cmd_tree - bind command parse tree. + bind_point - Policy bind point. + """ + if bind_point not in self._cmp_bind_info: + self._cmp_bind_info[bind_point] = OrderedDict() + self._cmp_bind_info[bind_point]["bind_parse_trees"] = [] + self._cmp_bind_info[bind_point]["is_classic_policy_bound"] = False + self._cmp_bind_info[bind_point]["is_advanced_policy_bound"] = False + + # -type keyword exists only for advanced policy + # bindings. + if bind_cmd_tree.keyword_exists("type"): + self._cmp_bind_info[bind_point]["is_advanced_policy_bound"] = True + else: + self._cmp_bind_info[bind_point]["is_classic_policy_bound"] = True + self._cmp_bind_info[bind_point]["bind_parse_trees"]. \ + append(bind_cmd_tree) + + def check_functionality(self): + """ + Handles if there are global bindings and + any vserver bindings. + Both classic and advanced policies can be bound + to CMP global. Choosing which set of policies to + evaluate depends on two factors. + 1. CMP global parameter + 2. policy type bound to vserver + First preference will be vserver, if vserver + exists and has classic policies bound to it, + then classic policy set is selected from + global and does not depend on global CMP parameter. + Same way if vserver has advanced policies, then + advanced policy set is selected from global. + If vserver does not exists, then depending on + global CMP parameter, policy set is choosen. + Functionality will change in the following + cases after conversion: + Global bindings and CMP parameter: + 1. When both classic and advanced policies + are bound to global. + 2. When classic policies are bound and + cmp parameter is advanced. + 3. When advanced policies are bound and + cmp parameter is classic. + Global and vserver bindings: + 4. When classic policies are bound to vserver and + advanced policies are bound to global. + 5. When advanced policies are bound to vserver and + classic policies are bound to global. + 6. When classic policies are bound to vserver and + both classic and advanced policies are bound to + global. + 7. When advanced policies are bound to vserver and + both classic and advanced policies are bound to + global. + Returns True if there is any conflict and functionality + will change, else returns False. + """ + # When both classic and advanced policies are + # bound to cmp global. This covers case 1,6,7 + # that are mentioned above. + global_bind_point = "" + if self._cmp_bind_info[global_bind_point][ + "is_classic_policy_bound"] and self. \ + _cmp_bind_info[global_bind_point]["is_advanced_policy_bound"]: + logging.error( + "Both classic and advanced policies " + "are bound to CMP global. Now classic policies are " + "converted to advanced. This will change the " + "functionality. CMP policy bindings are commented out. " + "Modify the bindings of CMP policies manually." + ) + return True + + conflict_exists = False + # When Global parameter and policies bound + # at global level does not match. + # This covers case 2 and 3 + policy_bound = '' + if self._cmp_bind_info[global_bind_point]["is_classic_policy_bound"]: + policy_bound = "classic" + else: + policy_bound = "advanced" + if not policy_bound == self._initial_cmp_parameter: + logging.error( + "There is a mismatch between global " + "parameter and policy type that are bound. " + "Now classic policies are converted to advanced " + "and cmp global parameter policy type is set to " + "advanced. This will change the functionality. " + "CMP policy bindings are commented out. Modify " + "the global bindings of CMP policies manually." + ) + conflict_exists = True + + # Both global and vserver bindings. + for bind_point in self._cmp_bind_info: + # case 4 and 5. + if bind_point == global_bind_point: + continue + if self._cmp_bind_info[global_bind_point][ + "is_classic_policy_bound"] and self. \ + _cmp_bind_info[bind_point]["is_advanced_policy_bound"]: + logging.error(( + "Classic policies are bound to cmp global " + "and advanced policies are bound to vserver {}." + " Now classic policies are converted to advanced. " + "This will change the functionality. CMP policy bindings " + "are commented out. Modify the bindings of CMP policies " + "manually.").format(bind_point) + ) + conflict_exists = True + elif self._cmp_bind_info[global_bind_point][ + "is_advanced_policy_bound"] and \ + self._cmp_bind_info[bind_point]["is_classic_policy_bound"]: + logging.error(( + "Advanced policies are bound to " + "cmp global and classic policies are bound " + "to vserver {}. Now classic policies are " + "converted to advanced. This will change the " + "functionality. CMP policy bindings are commented out. " + "Modify the bindings of CMP policies " + "manually.").format(bind_point) + ) + conflict_exists = True + return conflict_exists + + def global_binding_exists(self): + """ + Returns True if there is any CMP policy + bound to cmp global. + """ + return "" in self._cmp_bind_info + + def vserver_binding_exists(self): + """ + Returns True if there is any CMP policy + bound to any vserver. + """ + # CMP policy can be bound to global/ + # CS/CR/LB vservers. + for bind_point in self._cmp_bind_info: + if not bind_point == "": + return True + return False + + def resolve_cmp_param_global_binding(self): + """ + Comment the policy global bindings that do not + match the cmp parameter and throw warning. + Returns commented out bind command list. + """ + commented_bind_cmd_list = [] + global_bind_point = "" + if self._initial_cmp_parameter == "classic" and \ + self._cmp_bind_info[global_bind_point][ + "is_advanced_policy_bound"]: + # Comment the advanced policies that are bound. + logging.warning( + "Bindings of advanced CMP policies to cmp global " + "are commented out, because initial global cmp parameter " + "is classic but advanced policies are bound. Now global " + "cmp parameter policy type is set to advanced. If commands " + "are required please take a backup because comments will " + "not be saved in ns.conf after triggering 'save ns config'." + ) + # Iterate in reverse order, since we will be removing + # elements from list. + for tree in \ + reversed(self._cmp_bind_info[global_bind_point][ + "bind_parse_trees"]): + if tree.keyword_exists("type"): + bind_cmd = '#' + str(tree) + # Insert at 0, to preserve the order. + commented_bind_cmd_list.insert(0, bind_cmd) + self._cmp_bind_info[global_bind_point][ + "bind_parse_trees"].remove(tree) + elif self._initial_cmp_parameter == "advanced" and \ + self._cmp_bind_info[global_bind_point][ + "is_classic_policy_bound"]: + # Comment the classic policies that are bound. + logging.warning( + "Bindings of classic CMP policies to cmp global " + "are commented out, because initial global cmp parameter " + "is advanced but classic policies are bound. Now all " + "classic CMP policies are converted to advanced. If " + "commands are required please take a backup because comments " + "will not be saved in ns.conf after triggering " + "'save ns config'." + ) + # Iterate in reverse order, since we will be removing + # elements from list. + for tree in \ + reversed(self._cmp_bind_info[global_bind_point][ + "bind_parse_trees"]): + if not tree.keyword_exists("type"): + bind_cmd = '#' + str(tree) + # Insert at 0, to preserve the order. + commented_bind_cmd_list.insert(0, bind_cmd) + self._cmp_bind_info[global_bind_point][ + "bind_parse_trees"].remove(tree) + return commented_bind_cmd_list + + def is_same_policy_type(self): + """ + Returns True if all vservers have same + type of policies and matches with the + cmp parameter. + """ + global_bind_point = "" + if self._initial_cmp_parameter == "classic": + key = "is_advanced_policy_bound" + else: + key = "is_classic_policy_bound" + for bind_point in self._cmp_bind_info: + # skip for global bind point. + if bind_point == "": + continue + if self._cmp_bind_info[bind_point][key]: + return False + return True + + @common.register_for_final_call + def get_cmp_policy_bindings(self): + """ + Checks if the functionality will change + after conversion. If functionality will change, + returns all bind command parse trees(CMP + policy bindings) saved in _cmp_bind_info. + If functionality will not change, calls the + Binding infra for the priority analysis. + This should be called only at the end of + processing of entire ns.conf file. + Return value - list of parse trees. + """ + tree_list = [] + conflict_exists = False + + policy_type = self.__class__.__name__ + priority_arg = "priority" + goto_arg = "gotoPriorityExpression" + + if self.global_binding_exists() and not self.vserver_binding_exists(): + # Only global bindings. + # Comment policy bindings which do not match + # with cmp parameter. This will resolve the + # issue and there won't be any functionality + # change. + tree_list += self.resolve_cmp_param_global_binding() + elif self.global_binding_exists() and self.vserver_binding_exists(): + # Both vserver and global bindings. + if self.is_same_policy_type(): + # If all the vservers uses the same policy + # type and matches with the cmp parameter, + # comment the global bindings that do not match + # that policy type. + tree_list += self.resolve_cmp_param_global_binding() + else: + # If policy type is not same then + # check for the funtionality change. + conflict_exists = self.check_functionality() + + # If functionality will change, return bind commands + # without modifying priority and goto of bind commands. + if conflict_exists: + for bind_point in self._cmp_bind_info: + for tree in \ + self._cmp_bind_info[bind_point]["bind_parse_trees"]: + tree_list.append('#' + str(tree)) + else: + # If there is no policy type conflict, use Bind + # analysis infra for setting priority and goto. + module = self.__class__.__name__ + for bind_point in self._cmp_bind_info: + if bind_point == "": + for tree in \ + self._cmp_bind_info[bind_point]["bind_parse_trees"]: + if tree in self._classic_builtin_bind: + policy_name = self._classic_builtin_bind[tree] + else: + policy_name = tree.positional_value(0).value + tree_list += self.convert_global_bind( + tree, tree, policy_name, module, + priority_arg, goto_arg) + else: + for tree in \ + self._cmp_bind_info[bind_point]["bind_parse_trees"]: + if tree in self._classic_builtin_bind: + policy_name = self._classic_builtin_bind[tree] + else: + policy_name = tree.keyword_value( + "policyName")[0].value + tree_list += self.convert_entity_policy_bind( + tree, tree, policy_name, + policy_type, priority_arg, goto_arg) + return tree_list diff --git a/nspepi/nspepi2/convert_filter_command.py b/nspepi/nspepi2/convert_filter_command.py new file mode 100644 index 0000000..1bbd531 --- /dev/null +++ b/nspepi/nspepi2/convert_filter_command.py @@ -0,0 +1,955 @@ +#!/usr/bin/env python2 + +# Copyright 2021 Citrix Systems, Inc. All rights reserved. +# Use of this software is governed by the license terms, if any, +# which accompany or are included with this software. + +''' +Note: + 1. If action has value as prebody or postbody then there will not be any conversion + 2. If policy bind command has parameter "state disabled" then converted bind command will be commented out + 3. Filter feature of actionType FORWARD is not supported currently + 4. bind command of filter policies gets complicated if ns.conf already contains vserver of ptotocol type HTTP/S and rewrite/responder policy bindings: + 1. If goto is END/USE_INVOCATION_RESULT exists in existing rewrite local bindings and|not global bindings + 2. If goto is END/USE_INVOCATION_RESULT exists in existing rewrite global bindings and|not local bindings + Then comment out all partially converted rewrite global and local bindings otherwise do proper convertion. + Same applies for responder conditions. + +---INPUT WITH ADD ACTIONTYPE--- +add filter action act1 ADD "H1:Value" +add filter action act2 ADD "H1:::::Value" +add filter action act3 ADD H1::::::Value +add filter action act4 ADD H1: +add filter action act5 ADD H1:: +add filter action act6 ADD "H1:::" +add filter action act7 ADD "abcd:\"d1\"" +add filter action act8 ADD "abcd:::::::\"d1\"" +add filter action act10 ADD "@5%^&:Vl54" +add filter action act11 ADD q/\t:123/ +add filter action act12 ADD q/Name:\t/ +add filter action act13 ADD q/Name:\\t/ +add filter action act14 ADD prebody:123 +add filter action act15 ADD postbody:1234 +add filter action act16 ADD "\"H1:grv\"" +add filter action act17 ADD "/H1/:/d/1" +add filter action act18 ADD "/H1/:d/1" +add filter action act20 ADD "\H1:d1" +add filter action act21 add "/\H1:d1" +add filter action act22 ADD "\\\\H1:\\\d\\\\1\\\\2\\\\" +add filter action act23 add "H1:\\n" +add filter action act24 add "H\\n1:\\n" +add filter action act25 add "H \n1:\\n" +add filter action act26 add "H1:Value1:Value2" +add filter action act27 add "H1:%%HTTP.TRANSID%%" +add filter action act28 add prebody +add filter action act29 add postbody +add filter policy add_pol1_1 -rule ns_true -resAction act1 +add filter policy add_pol1_2 -rule ns_true -reqAction act1 +add filter policy add_pol2_1 -rule ns_true -resAction act27 +add filter policy add_pol2_2 -rule ns_true -reqAction act27 +add filter policy add_pol3_1 -rule ns_true -reqAction act28 +add filter policy add_pol3_2 -rule ns_true -resAction act28 +add filter policy add_pol4_1 -rule ns_true -reqAction act29 +add filter policy add_pol4_2 -rule ns_true -resAction act29 +bind filter global add_pol1_1 [-priority ]) +bind filter global add_pol1_2 [-priority ]) +bind lb vserver -policyName add_pol1_1 [-priority ]) +bind lb vserver -policyName add_pol1_2 [-priority ]) +bind cs vserver -policyName add_pol1_1 [-priority ]) +bind cs vserver -policyName add_pol1_2 [-priority ]) +bind cr vserver -policyName add_pol1_1 [-priority ]) +bind cr vserver -policyName add_pol1_2 [-priority ]) +---CONVERTED RESULT--- +add rewrite action act1 insert_http_header f_H1 "\"Value\"" +add rewrite action act2 insert_http_header H1 "\"::::Value\"" +add rewrite action act3 insert_http_header H1 "\":::::Value\"" +add rewrite action act4 insert_http_header H1 "\"\"" +add rewrite action act5 insert_http_header H1 "\":\"" +add rewrite action act6 insert_http_header H1 "\"::\"" +add rewrite action act7 insert_http_header abcd "\"\\\"d1\\\"\"" +add rewrite action act8 insert_http_header abcd "\"::::::\\\"d1\\\"\"" +add rewrite action act10 insert_http_header @5%^& "\"Vl54\"" +add rewrite action act11 insert_http_header "\\t" "\"123\"" +add rewrite action act12 insert_http_header Name "\"\\\\t\"" +add rewrite action act13 insert_http_header Name "\"\\\\\\\\t\"" +add rewrite action act14 insert_http_header prebody "\"123\"" +add rewrite action act15 insert_http_header postbody "\"1234\"" +add rewrite action act16 insert_http_header "\"H1" "\"grv\\\"\"" +add rewrite action act17 insert_http_header /H1/ "\"/d/1\"" +add rewrite action act18 insert_http_header /H1/ "\"d/1\"" +add rewrite action act20 insert_http_header "\\H1" "\"d1\"" +add rewrite action act21 insert_http_header "/\\H1" "\"d1\"" +add rewrite action act22 insert_http_header "\\\\H1" + "\"\\\\\\\\d\\\\\\\\1\\\\\\\\2\\\\\\\\\"" +add rewrite action act23 insert_http_header H1 "\"\\\\n\"" +add rewrite action act24 insert_http_header "H\\n1" "\"\\\\n\"" +add rewrite action act25 insert_http_header "H \n1" "\"\\\\n\"" +add rewrite action act26 insert_http_header H1 "\"Value1:Value2\"" +add rewrite action act27 insert_http_header H1 HTTP.REQ.TXID +add rewrite action nspepi_adv_act27 insert_http_header H1 HTTP.RES.TXID +add filter action act28 add prebody +add filter action act29 add postbody +add rewrite policy add_pol1_1 TRUE act1 +add rewrite policy add_pol1_2 TRUE act1 +add rewrite policy add_pol2_1 TRUE nspepi_adv_act27 +add rewrite policy add_pol2_2 TRUE act27 +add filter policy add_pol3_1 -rule ns_true -reqAction act28 +add filter policy add_pol3_2 -rule ns_true -resAction act28 +add filter policy add_pol4_1 -rule ns_true -reqAction act29 +add filter policy add_pol4_2 -rule ns_true -resAction act29 +bind rewrite global add_pol1_1 NEXT -TYPE RES_DEFAULT +bind rewrite global add_pol1_2 NEXT -TYPE REQ_DEFAULT +bind lb vserver -policyName add_pol1_1 [-priority ]) -gotoPriorityExpression NEXT -TYPE RESPONSE +bind lb vserver -policyName add_pol1_2 [-priority ]) -gotoPriorityExpression NEXT -TYPE REQUEST +bind cs vserver -policyName add_pol1_1 [-priority ]) -gotoPriorityExpression NEXT -TYPE RESPONSE +bind cs vserver -policyName add_pol1_2 [-priority ]) -gotoPriorityExpression NEXT -TYPE REQUEST +bind cr vserver -policyName add_pol1_1 [-priority ]) -gotoPriorityExpression NEXT -TYPE RESPONSE +bind cr vserver -policyName add_pol1_2 [-priority ]) -gotoPriorityExpression NEXT -TYPE REQUEST + +---INPUT WITH CORRUPT ACTIONTYPE--- +add filter action act30 CORRUPT TEST_HEADER +add filter policy corrupt_pol1_1 -rule ns_true -reqAction act30 +add filter policy corrupt_pol1_2 -rule ns_true -resAction act30 +bind filter global corrupt_pol1_1 [-priority ]) +bind filter global corrupt_pol1_2 [-priority ]) +bind vserver -policyName corrupt_pol1_1 [-priority ]) +bind vserver -policyName corrupt_pol1_2 [-priority ]) +---CONVERTED RESULT--- +add rewrite action act30 corrupt_http_header TEST_HEADER +add rewrite policy corrupt_pol1_1 TRUE act30 +add rewrite policy corrupt_pol1_2 TRUE act30 +bind rewrite global corrupt_pol1_1 NEXT -TYPE REQ_DEFAULT +bind rewrite global corrupt_pol1_2 NEXT -TYPE RES_DEFAULT +bind lb vserver -policyName corrupt_pol1_1 [-priority ]) -gotoPriorityExpression NEXT -TYPE REQUEST +bind lb vserver -policyName corrupt_pol1_2 [-priority ]) -gotoPriorityExpression NEXT -TYPE RESPONSE + +---INPUT WITH ERRORCODE ACTIONTYPE--- +add filter action act1 ERRORCODE 200 "Good URL" +add filter action act2 ERRORCODE 200 +add filter policy error_pol1 -rule ns_true -resAction act1 +add filter policy error_pol2 -rule ns_true -reqAction act1 +add filter policy error_pol3 -rule ns_true -resAction act2 +bind filter global error_pol1 [-priority ]) +bind filter global error_pol2 [-priority ]) +bind vserver -policyName error_pol1 [-priority ]) +bind vserver -policyName error_pol2 [-priority ]) +---CONVERTED RESULT--- +add responder action act1 respondwith "HTTP.REQ.VERSION.APPEND(\" 200 OK\r\n + Connection: close\r\nContent-Length: 21\r\n\r\nGood URL\")" +add rewrite action nspepi_adv_act1 replace_http_res "HTTP.REQ.VERSION.APPEND + (\" 200 OK\r\nConnection: close\r\nContent-Length: 21\r\n\r\n + Good URL\")" +add responder action act2 respondwith "HTTP.REQ.VERSION.APPEND(\" 200 OK\r\n + Connection: close\r\nContent-Length: 0\r\n\r\n\")" +add rewrite action nspepi_adv_act2 replace_http_res "HTTP.REQ.VERSION.APPEND + (\" 200 OK\r\nConnection: close\r\nContent-Length: 0\r\n\r\n\")" +add rewrite policy error_pol1 TRUE nspepi_adv_act1 +add responder policy error_pol2 TRUE act1 +add rewrite policy error_pol3 TRUE nspepi_adv_act2 +bind rewrite global error_pol1 NEXT -TYPE RES_DEFAULT +bind rewrite global error_pol2 NEXT -TYPE REQ_DEFAULT +bind vserver -policyName error_pol1 [-priority ]) -gotoPriorityExpression NEXT -TYPE RESPONSE +bind vserver -policyName error_pol2 [-priority ]) -gotoPriorityExpression END -TYPE REQUEST + +---INPUT WITH DROP ACTION--- +add filter action act1 DROP +add filter policy pol1 -rule "ns_true" -reqAction act1 +add filter policy pol2 -rule "ns_true" -resAction act2 +add filter policy pol3 -rule ns_true -reqAction DROP +add filter policy pol4 -rule ns_true -resAction DROP +bind filter global pol1 [-priority ]) +bind filter global pol2 [-priority ]) +bind vserver -policyName pol1 [-priority ]) +bind vserver -policyName pol2 [-priority ]) +---CONVERTED RESULT--- +add responder policy pol1 TRUE DROP +add rewrite policy pol2 TRUE DROP +add responder policy pol3 TRUE DROP +add rewrite policy pol4 TRUE DROP +bind responder global pol1 END -type REQ_DEFAULT +bind rewrite global pol2 NEXT -type RES_DEFAULT +bind vserver -policyName pol1 [-priority ]) -gotoPriorityExpression NEXT -TYPE REQUEST +bind vserver -policyName pol2 [-priority ]) -gotoPriorityExpression NEXT -TYPE RESPONSE + +---INPUT WITH RESET ACTION--- +add filter action act1 RESET +add filter policy pol1 -rule "ns_true" -reqAction act1 +add filter policy pol2 -rule "ns_true" -resAction act1 +add filter policy pol3 -rule ns_true -reqAction RESET +add filter policy pol4 -rule ns_true -resAction RESET +bind filter global pol1 [-priority ]) +bind filter global pol2 [-priority ]) +bind vserver -policyName pol1 [-priority ]) +bind vserver -policyName pol2 [-priority ]) +---CONVERTED RESULT--- +add responder policy pol1 TRUE RESET +add rewrite policy pol2 TRUE RESET +add responder policy pol3 TRUE RESET +add rewrite policy pol4 TRUE RESET +bind responder global pol1 END -type REQ_DEFAULT +bind rewrite global pol2 NEXT -type RES_DEFAULT +bind vserver -policyName pol1 [-priority ]) -gotoPriorityExpression NEXT -TYPE REQUEST +bind vserver -policyName pol2 [-priority ]) -gotoPriorityExpression NEXT -TYPE RESPONSE + +# Remove this comment line at the time of transforming actionType FORWARD +---INPUT FORWARD ACTIONTYPE--- +add filter action act1 FORWARD serviceName +add filter policy pol1 -rule "ns_true" -reqAction act1 +bind filter global pol1 [-priority ]) +bind vserver -policyName pol1 [-priority ]) +---CONVERTED RESULT--- +add filter action act1 FORWARD serviceName +add filter policy pol1 -rule "ns_true" -reqAction act1 +bind filter global pol1 [-priority ]) +bind vserver -policyName pol1 [-priority ]) +''' + +import re +import logging +import copy +import nspepi_common as common +from nspepi_parse_tree import * +from convert_classic_expr import * +from collections import OrderedDict +import convert_cli_commands as cli_cmds +from convert_lb_cmd import * +from itertools import chain +from convert_responder_command import Responder +from convert_rewrite_command import Rewrite + + +# All module names starting with "convert_" are parsed to detect and register +# class methods +@common.register_class_methods +class CLITransformFilter(cli_cmds.ConvertConfig): + """ + Converts classic filter feature except htmlInjection and FORWARD type + """ + flow_type_direction_default = None + + def __init__(self): + """ + _action_command - Dictionary to store action name and converted classic + action commands of type 'errorcode' and 'add' + _vserverName_list - List to store vserver names + _actionTypeName - Dictionary to store + {actionTypes: list of action names} + _converted_pol_param - Dictionary to store- policyName: [re[sq]Action, + converted module] + _policy_command - List to store converted actions which are called + by policy and policy commands itself for those which + has two converted actions for each classic action. + Otherwise store just converted policy commands + _htmlInjection - List to store action names of those actions which + points to html injection values: prebody or postbody + _bind_tree_rw - List of partially converted rewrite bind commands + to help in commenting out only converted rewrite binds if + existing rewrite has END/USE_INVOCATION_RESULT + _bind_tree_resp - List of partially converted responder bind commands + to help in commenting out only converted responder binds + if existing responder has END/USE_INVOCATION_RESULT + """ + self._action_command = OrderedDict() + self._actionTypeName = OrderedDict([ + ("reset", ["reset"]), ("drop", ["drop"])]) + self._converted_pol_param = OrderedDict() + self._policy_command = [] + self._htmlInjection = [] + self._bind_tree_rw = [] + self._bind_tree_resp = [] + ''' + # TODO - This should be uncommented at the time of + # transformation for FORWARD actionType. + self._vserverName_list = set([]) + ''' + @common.register_for_cmd("add", "filter", "action") + def convert_filter_action(self, action_parse_tree): + """ + Transform classic feature for filter Action commands + In case of actionType DROP and RESET: + 1. Store to be used in transformed filter + policy with action as + 2. Remove "add filter action " + as this is not to be transformed to rewrite/responder, + Only its actionType to be used in transformed policy. + + Information for variables used here: + original_cmd - keeps the copy of classic command + action_type - type of classic action + actionName - name of original action + header_value - refers to the value of Header when + action type is ADD + Filter_variable - htmlInjection variable as key-value + pair example-{variable: [list of advance exp]} + html_page - HTML page when action type is ERRORCODE + content_length - indicates Content-Length + """ + original_cmd = copy.deepcopy(action_parse_tree) + # Initialize tree to empty list. + action_parse_tree_list = [] + action_type = original_cmd.positional_value(1).value.lower() + if (action_type in [ + "add", "corrupt", "errorcode", "reset", "drop", "forward"]): + """ Check if original command has filter group """ + + # Store actionType and actionName + actionName = original_cmd.positional_value( + 0).value + if action_type not in self._actionTypeName: + self._actionTypeName[action_type] = [] + self._actionTypeName[action_type].append(actionName) + + if (action_type == "add"): + """ + Transformation for classic command of actionType ADD + KEY_POINTS: + 1. Store input action name and list of 2 converted actions + for single input, only if input has html injection + variable as a value + 2. Append new action name nspepi_adv_ + with input action name in actionTypeName dictionary + for above condition + 3. return input command only if value in input is + prebody or postbody + """ + header_value = original_cmd.positional_value(2).value + if not ((header_value == "postbody") or ( + header_value == "prebody")): + # Transformation for filter action of ADD as actionType + # if value is not pre/postbody + action_parse_tree = CLICommand("add", "rewrite", "action") + action_name = CLIPositionalParameter(actionName) + action_type_adv = CLIPositionalParameter( + "insert_http_header") + action_parse_tree.add_positional_list([ + action_name, action_type_adv]) + + # Splitting classic action cmd value from 1st colon(:) + match = header_value.split(":", 1) + header = match[0] + value = match[1] + headerName = CLIPositionalParameter(header) + Filter_variable = { + 'HTTP.TRANSID': ['HTTP.REQ.TXID', 'HTTP.RES.TXID'], + 'HTTP.XID': ['HTTP.REQ.TXID', 'HTTP.RES.TXID']} + + # Checks for Value/stringBuildExpression that has + # Html Injection Variable surrounded by %% + match_value = re.search(r"%%(.*)%%", value) + if not match_value: + stringBuildExp = CLIPositionalParameter( + action_parse_tree.normalize(value, True)) + action_parse_tree.add_positional_list( + [headerName, stringBuildExp]) + action_parse_tree.set_upgraded() + return [action_parse_tree] + else: + # if value of header is a variable surrounded between + # %%%%, then value is considered as HTMLInjection + # variable. That variable can be used in request and + # response sides, and value is generated based on the + # side. But, in the advanced policy expression, there are + # different expressions for the different sides. So, + # for each side an action needs to be created. + # rewrite_action_response: action which has HTTP.RES.XX + # action_parse_tree: action which has HTTP.REQ.XX + + # Save action name in a list which is used to + # identify an action command + if actionName not in self._action_command: + self._action_command[actionName] = [] + rewrite_action_response = copy.deepcopy( + action_parse_tree) + if match_value.group(1) in Filter_variable: + # Add a new rewrite action of different name whose + # stringBuildExpr should be expression of + # RESPONSE side + rewrite_action_response.positional_value( + 0).set_value("nspepi_adv_" + actionName) + stringBuildExp_res = CLIPositionalParameter( + rewrite_action_response.normalize( + Filter_variable[match_value.group(1)][1])) + rewrite_action_response.add_positional_list( + [headerName, stringBuildExp_res]) + rewrite_action_response.set_upgraded() + + # Modify current rewrite action where + # stringBuildExpr should be of REQUEST side + stringBuildExp_req = CLIPositionalParameter( + action_parse_tree.normalize( + Filter_variable[match_value.group(1)][0])) + action_parse_tree.add_positional_list( + [headerName, stringBuildExp_req]) + action_parse_tree.set_upgraded() + + # Store actionName, and converted command + self._action_command[actionName].append( + action_parse_tree) + self._action_command[actionName].append( + rewrite_action_response) + + # Store actionType and actionName + self._actionTypeName[action_type].append( + rewrite_action_response.positional_value( + 0).value) + return [] + else: + # Prebody/postbody value indicates that this config is + # being used for HTMLinjection feature, and conversion + # of HTMLinjection config is not supported, so return + # the input command as the output. + + # Remove action name from stored dictionary. + self._actionTypeName[action_type].remove(actionName) + + # Store list of corresponding action names + self._htmlInjection.append(actionName) + action_parse_tree_list = [original_cmd] + logging.error( + "Conversion of [{}] not supported in this tool." + "".format(str(original_cmd).strip())) + elif (action_type == "corrupt"): + """ + Transformation for filter action of CORRUPT as actionType + """ + action_parse_tree = CLICommand("add", "rewrite", "action") + action_name = CLIPositionalParameter(actionName) + action_type_adv = CLIPositionalParameter("corrupt_http_header") + target = CLIPositionalParameter( + original_cmd.positional_value(2).value) + action_parse_tree.add_positional_list([ + action_name, action_type_adv, target]) + action_parse_tree.set_upgraded() + action_parse_tree_list = [action_parse_tree] + elif (action_type == "errorcode"): + """ + Transformation for filter action of ERRORCODE as actionType. + This transforms filter action to advanced command of + both rewrite and responder feature. Later one of transformed + command which is not used by policy should be removed during + cleanup process + KEY-POINTS: + single input will be converted to 2 actions, one will have + nspepi_adv_ prefixed to name under rewrite module and + second will have its original name under responder module + """ + # Save action name in a list which is used to + # identify an action command + if actionName not in self._action_command: + self._action_command[actionName] = [] + # A. Parse tree for advance command with responder feature + responder_action = CLICommand("add", "responder", "action") + action_name = CLIPositionalParameter(actionName) + action_type_adv = CLIPositionalParameter("respondwith") + responder_action.add_positional_list([ + action_name, action_type_adv]) + responder_action.set_upgraded() + + # B. Parse tree for advanced command with rewrite feature + rewrite_action = CLICommand("add", "rewrite", "action") + action_name = CLIPositionalParameter( + "nspepi_adv_"+original_cmd.positional_value(0).value) + action_type_adv = CLIPositionalParameter("replace_http_res") + rewrite_action.add_positional_list([ + action_name, action_type_adv]) + rewrite_action.set_upgraded() + + # C. Assignment - common to both rewrite and responder + status_code = original_cmd.positional_value(2).value + status_list = { + '100': 'CONTINUE', + '200': 'OK', + '201': 'CREATED', + '202': 'ACCEPTED', + '203': 'Non-Authoritative', + '204': 'No Content', + '205': 'Reset Content', + '206': 'Partial Content', + '300': 'Multiple Choices', + '301': 'Moved Permanently', + '302': 'FOUND', + '303': 'See Other', + '304': 'Not Modified', + '305': 'Use Proxy', + '400': 'BAD REQUEST', + '401': 'Unauthorized', + '402': 'Payment Required', + '403': 'FORBIDDEN', + '404': 'Not Found', + '405': 'Method Not Allowed', + '406': 'Not Acceptable', + '408': 'Request Timeout', + '407': 'Proxy Authentication Required', + '409': 'Conflict', + '410': 'Gone', + '411': 'Length Required', + '412': 'Precondition Failed', + '413': 'Request Entity Too Large', + '414': 'Request-URI Too Long', + '415': 'Unsupported Media Type', + '500': 'INTERNAL SERVER ERROR', + '501': 'Not Implemented', + '502': 'Bad Gateway', + '503': 'SERVICE UNAVALIABLE', + '504': 'Gateway Timeout', + '505': 'HTTP Version Not Supported' + } + if status_code not in status_list: + status_message = "Status " + str(status_code) + else: + status_message = status_list[status_code] + html_page = "" + + # 1. When html content/body is present in classic command: + if original_cmd.positional_value(3): + html_page = original_cmd.positional_value( + 3).value + + # Set content Length + content_length = len(html_page) + html_text = ' ' + status_code + ' ' + status_message \ + + '\r\nConnection: close\r\nContent-Length: '\ + + str(content_length) + '\r\n\r\n'\ + + html_page + target_value = 'HTTP.REQ.VERSION.APPEND(' \ + + responder_action.normalize(html_text) \ + + ')' + + # A.1. Transformation to Responder feature + target = CLIPositionalParameter(target_value) + responder_action.add_positional(target) + responder_action.set_upgraded() + + # B.1. Transformation to rewrite feature + rewrite_action.add_positional(target) + rewrite_action.set_upgraded() + + # Store actionName, and converted command + self._action_command[actionName].append(responder_action) + self._action_command[actionName].append(rewrite_action) + + # Store actionType and actionName + self._actionTypeName[action_type].append(str( + rewrite_action.positional_value(0).value)) + return [] + elif (action_type == "drop") or (action_type == "reset"): + """ Transformation for classicAction DROP/RESET actionType + The action command should be removed and its actionType should + be used while transforming filterpolicy command. + Example: + add filter action act1 DROP/RESET + Conversion Process: + Return null for classic action command transformed. + """ + # Save action name in a list which is used to + # identify an action command + return [] + elif action_type == "forward": + """Transformation for filter action command of FORWARD as + actionType, LBvserver is created and bound to the existing + service via VserverCreation function and LBvserver Name is + fetched and used in transformed command of feature group CS + vs_kw - key for targetLBVserver + lb_name - new lb vserver to be used by cs action + """ + ''' + # TODO - there are issues with FORWARD actionType in binding, + # so we won't currently enable conversions for filter of + # FORWARD actionType. + service = original_cmd.positional_value(2).value + lb_name = "nspepi_lb_vserver_" + service + action_parse_tree = CLICommand("add", "cs", "action") + action_name = CLIPositionalParameter(actionName) + action_parse_tree.add_positional(action_name) + vs_kw = CLIKeywordParameter(CLIKeywordName("targetLBVserver")) + vs_kw.add_value(lb_name) + action_parse_tree.add_keyword(vs_kw) + action_parse_tree.set_upgraded() + if lb_name in self._vserverName_list: + # If vserverName already exists in the list + # then just output the transformed action + self._vserverName_list.add(lb_name) + action_parse_tree_list = [action_parse_tree] + else: + # If vserverName not in the list + # then output- add vserver, bind vserver and + # transformed action + self._vserverName_list.add(lb_name) + + # Add LB Vserver with name nspepi_lb_vserver_ + vserver = CLICommand("add", "lb", "vserver") + vs_name = CLIPositionalParameter(lb_name) + vs_type = CLIPositionalParameter("http") + vserver.add_positional_list([vs_name, vs_type]) + vserver.set_upgraded() + + # Bind LB Vserver with Service Name which is available in + # classic action command + bindVserver = CLICommand("bind", "lb", "vserver") + bind_vs_name = CLIPositionalParameter(str(vs_name)) + bind_service_name = CLIPositionalParameter(service) + bindVserver.add_positional_list([ + bind_vs_name, bind_service_name]) + bindVserver.set_upgraded() + action_parse_tree_list = [ + vserver, bindVserver, action_parse_tree] + ''' + action_parse_tree_list = [original_cmd] + logging.error( + "Conversion of [{}] not supported in this tool." + "".format(str(original_cmd).strip())) + + else: + """ + If originial command does not have any one of the filter + action type: ADD, ERRORCODE, DROP, RESET, CORRUPT, and FORWARD + then log the error message + """ + action_parse_tree_list = [original_cmd] + logging.error( + 'Error in converting original command since' + + ' CLI context of filter feature is invalid: ' + + '{}'.format(str(original_cmd))) + return action_parse_tree_list + + @common.register_for_cmd("add", "filter", "policy") + def convert_filter_policy(self, policy_parse_tree): + """ + Transform classic feature for filter policy commands + KEY-POINTS - + 1) nspepi_adv_ stored in self._actionTypeName will + point to REWRITE module always + 2) Store policy name, and tuple of re[sq]Action key name, converted + module to help in binding time. During binding, + feature group and bind point are keys to focus. + Information for variables used here: + policyName - Name of policy + new_policy - function to return parse tree of the converted command + policy_action - action called by policy + converted_pol_cmd - tree for converted input command + dict_key - stored action type + dict_value - stored list of action names + action_tree - stored action command at respective index in + _action_command + """ + original_cmd = copy.deepcopy(policy_parse_tree) + policyName = policy_parse_tree.positional_value(0).value + pol_obj = common.Policy(policyName, self.__class__.__name__, "classic") + common.pols_binds.store_policy(pol_obj) + # Convert classic expression + policy_parse_tree = CLITransformFilter.convert_keyword_expr( + policy_parse_tree, 'rule') + if not policy_parse_tree.upgraded: + return self.return_original_input(original_cmd, pol_obj) + converted_pol_cmd, policy_action_key, policy_action = self.new_policy( + policy_parse_tree, policyName) + if policy_action in self._htmlInjection: + # Return input for those policies which are calling actions + # having value as prebody or postbody Since they belong to + # html injection family + return self.return_original_input(original_cmd, pol_obj) + for dict_key, dict_value in self._actionTypeName.iteritems(): + """ Extract key and value from stored _actionTypeName + dictionary through action convertion """ + if policy_action not in dict_value: + continue + elif ("nspepi_adv_" + policy_action in dict_value): + """ + If input policy calls the action which + should point to the action either nspepi_adv_ or + original input name then below parameter should be + changed: + 1) Group should be REWRITE for ERRORCODE for resAction + and RESPONDER for reqAction. REWRITE for ADD on + either case. + 2) Action name should be "nspepi_adv_" for + resAction else original name + 3) Converted action should be taken is: + action of name nspepi_adv_ for resAction + else action of original name + """ + if (policy_action_key == "resAction"): + converted_pol_cmd.positional_value(2) \ + .set_value("nspepi_adv_" + policy_action) + if (dict_key == "errorcode") and \ + (policy_action_key == "reqAction"): + converted_pol_cmd.group = "responder" + # Store converted action command first to be + # returned in order + action_tree = (self._action_command[policy_action][1] if ( + policy_action_key == "resAction") else + self._action_command[policy_action][0]) + if action_tree not in self._policy_command: + # To avoid duplication + self._policy_command.append(action_tree) + break + elif (dict_key == "add") or (dict_key == "corrupt"): + # If input policy calls one-to-one action of ADD and + # CORRUPT type + break + elif (dict_key in ["drop", "reset"]): + # If input policy calls the custom action for drop and reset + if (policy_action_key == "reqAction"): + converted_pol_cmd.group = "responder" + policy_action = dict_key.upper() + converted_pol_cmd.positional_value(2) \ + .set_value(policy_action) + break + elif (dict_key == "forward"): + """ If input policy calls the action which points to the action + for FORWARD """ + # TODO - This should be uncommented at the time of + # transformation for FORWARD actionType. + """ + converted_pol_cmd = CLICommand("add", "cs", "policy") + policy_name = CLIPositionalParameter(policyName) + converted_pol_cmd.add_positional(policy_name) + rule = CLIKeywordParameter(CLIKeywordName("rule")) + rule.add_value(advanced_expr) + converted_pol_cmd.add_keyword(rule) + action_key = CLIKeywordParameter(CLIKeywordName("action")) + action_key.add_value(action_name) + converted_pol_cmd.add_keyword(action_key) + """ + # TODO - put it outside at the time of transformation for + # FORWARD actionType and return converted output. + + # To store policy name and tuple of reqAction key name and + # original module + return self.return_original_input(original_cmd, pol_obj) + else: + return self.return_original_input(original_cmd, pol_obj) + + # Changing the module name to converted module name + pol_obj.module = self.__class__.__name__ + # Store converted tree + self._policy_command.append(converted_pol_cmd) + # store policy name, tuple of re[?]Action key name + # and converted module + if policyName not in self._converted_pol_param: + self._converted_pol_param[policyName] = [] + self._converted_pol_param[policyName] += ( + policy_action_key, converted_pol_cmd.group) + return [] + + def new_policy(self, policy_parse_tree, policyName): + """ + This will return parse tree of the converted command + Information for arguments used here: + policy_parse_tree - parse tree of the command with + converted rule + policyName - Name of the policy + Information for variable used here: + policy_action_key - re[qs]Action key used in parse tree + """ + converted_pol_cmd = CLICommand("add", "rewrite", "policy") + + # To get the action used in policy + if policy_parse_tree.keyword_exists("reqAction"): + policy_action_key = "reqAction" + if policy_parse_tree.keyword_exists("resAction"): + policy_action_key = "resAction" + policy_action = policy_parse_tree.keyword_value( + policy_action_key)[0].value.lower() + advanced_expr = policy_parse_tree.keyword_value("rule")[0].value + policy_name = CLIPositionalParameter(policyName) + rule = CLIPositionalParameter(advanced_expr) + action_name = CLIPositionalParameter(policy_action) + converted_pol_cmd.add_positional_list([ + policy_name, rule, action_name]) + converted_pol_cmd.set_upgraded() + return converted_pol_cmd, policy_action_key, policy_action + + def return_original_input(self, original_cmd, pol_obj): + """ Return original input """ + logging.error( + "Conversion of [{}] not supported in this tool." + "".format(str(original_cmd).strip())) + # Setting policy type to advanced to avoid conversion + # during bind command + pol_obj.policy_type = "advanced" + return [original_cmd] + + @common.register_for_bind(["LB", "ContentSwitching", "CacheRedirection"]) + def convert_filter_vserver_bindings( + self, bind_parse_tree, policy_name, priority_arg, goto_arg): + """ + Handles converted policy bindings to vservers - LB, CS, CR + Syntax for converted policy binding: + bind lb/cr/cs vserver -policyName + [-priority ] + policy_type - type of policy from convert_filter_policy + If policy type during policy conversion is marked as + advanced (for FORWARD and htmlInjection type) then return input + Add -type explicitly + """ + policy_type = common.pols_binds.policies[policy_name].policy_type + if policy_type == "advanced": + # Return same if policy_type is marked advanced + return [bind_parse_tree] + flow_type = ("RESPONSE" if (self._converted_pol_param[ + policy_name][0] == "resAction") else "REQUEST") + bind_type = CLIKeywordParameter(CLIKeywordName("type")) + bind_type.add_value(str(flow_type)) + bind_parse_tree.add_keyword(bind_type) + bind_parse_tree.set_upgraded() + if self._converted_pol_param[policy_name][1] == "responder": + # Responder - Store partially converted tree separately + self._bind_tree_resp.append(bind_parse_tree) + else: + # Rewrite - Store partially converted tree separately + self._bind_tree_rw.append(bind_parse_tree) + return [] + + @common.register_for_cmd("bind", "filter", "global") + def convert_filter_global_bindings(self, bind_parse_tree): + """ + Handles global filter policy bindings. + Syntax for classic policy binding: + bind filter global [-priority ] + [-state (ENABLED | DISABLED)] + When classic filter policy is bound: + 1. If -state is DISABLED, comment the bind command. + 2. Add -type RE[QS]_DEFAULT keyword + 3. Throw error when functionality may change. + 4. Add as NEXT for rewrite policy bindings and + Add -gotoPriorityExpression as END for responder policy bindings + """ + orig_tree = copy.deepcopy(bind_parse_tree) + if bind_parse_tree.keyword_exists("state") and \ + bind_parse_tree.keyword_value("state")[0].value.lower() == \ + "disabled": + logging.warning(( + "Following bind command is commented out because" + " state is disabled. If command is required please take" + " a backup because comments will not be saved in ns.conf" + " after triggering 'save ns config': {}"). + format(str(bind_parse_tree).strip()) + ) + return ["#" + str(bind_parse_tree)] + policy_name = bind_parse_tree.positional_value(0).value + policy_type = common.pols_binds.policies[policy_name].policy_type + if policy_type == "advanced": + # Return input if policy_type is marked with "advanced" + return [orig_tree] + bind_parse_tree = CLICommand("bind", "rewrite", "global") + bind_parse_tree.original_line = str(orig_tree) + policyName = CLIPositionalParameter(policy_name) + bind_parse_tree.add_positional(policyName) + if orig_tree.keyword_exists("priority"): + priority = CLIPositionalParameter(orig_tree.keyword_value( + "priority")[0].value) + bind_parse_tree.add_positional(priority) + if self._converted_pol_param[policy_name][1] == "responder": + """ + if converted policy is of RESPONDER module then converted bind tree + should be of RESPONDER module else REWRITE in global binding. + """ + bind_parse_tree.group = "responder" + flow_type = ("RES_DEFAULT" if self._converted_pol_param[ + policy_name][0] == "resAction" else "REQ_DEFAULT") + bind_type = CLIKeywordParameter(CLIKeywordName("type")) + bind_type.add_value(str(flow_type)) + bind_parse_tree.add_keyword(bind_type) + bind_parse_tree.set_upgraded() + if bind_parse_tree.group == 'responder': + self._bind_tree_resp.append(bind_parse_tree) + else: + self._bind_tree_rw.append(bind_parse_tree) + return [] + + @common.register_for_final_call + def get_converted_cmds(self): + """ + Returns list of converted commands irrespective to whether + any action used in policy + Puts the converted command in order of first action then policy + converted_list - Returns tree of ordered action, policy and bind + Comment out converted bind command if existing rewrite/responder + policies are bound with GOTO either END or USE_INVOCATION_RESULT + """ + converted_list = [] + + # If no policy but only actions are in ns.conf, then just return those + # converted actions. + for act_name in self._action_command: + if (self._action_command[act_name][0] not in self._policy_command)\ + and (self._action_command[ + act_name][1] not in self._policy_command): + converted_list += self._action_command[act_name] + for converted_tree in self._policy_command: + # Ordering in a way of action first and then policy + if converted_tree.ot == 'action': + converted_list.insert(0, converted_tree) + else: + converted_list.append(converted_tree) + # Important points for Bind command conversions: + # bind command of filter policies gets complicated if ns.conf already contains vserver of HTTP/S protocol type and + # rewrite/responder policy bindings + # 1. If goto is END/USE_INVOCATION_RESULT exists in existing rewrite local bindings and|not global bindings + # 2. If goto is END/USE_INVOCATION_RESULT exists in existing rewrite global bindings and|not local bindings + # Then comment out all partially converted rewrite global and local bindings otherwise do proper convertion + # Same applies for responder conditions + rewrite_class = Rewrite() + responder_class = Responder() + position = "after" + vs_name = '' + for rw in self._bind_tree_rw: + if rw.ot == "global": + policy_name = rw.positional_value(0).value + priority_arg = 1 + goto_arg = 2 + if rw.ot == "vserver": + vs_name = rw.positional_value(0).value + policy_name = rw.keyword_value("policyName")[0].value + priority_arg = "priority" + goto_arg = "gotoPriorityExpression" + module = "Rewrite" + cli_cmds.ConvertConfig.bind_default_goto = "NEXT" + if (rewrite_class.rw_global_goto_exists == True) or ( + rewrite_class.rw_vserver_goto_exists == True): + bind_cmd = self.return_bind_cmd_warning(rw) + converted_list.append(bind_cmd) + else: + self.complete_convert_bind_cmd( + rw, policy_name, module, priority_arg, + goto_arg, position) + for resp in self._bind_tree_resp: + if resp.ot == "global": + policy_name = resp.positional_value(0).value + priority_arg = 1 + goto_arg = 2 + if resp.ot == "vserver": + policy_name = resp.keyword_value("policyName")[0].value + vs_name = resp.positional_value(0).value + priority_arg = "priority" + goto_arg = "gotoPriorityExpression" + module = "Responder" + cli_cmds.ConvertConfig.bind_default_goto = "END" + if (responder_class.resp_global_goto_exists == True) or ( + responder_class.resp_vserver_goto_exists == True): + bind_cmd = self.return_bind_cmd_warning(resp) + converted_list.append(bind_cmd) + else: + self.complete_convert_bind_cmd( + resp, policy_name, module, priority_arg, + goto_arg, position) + return converted_list + + def return_bind_cmd_warning(self, cmd): + # Return warnings and partially converted commented out bind command + logging.error("In ns.conf, existing advanced feature policies's bind commands have" + " gotoPriorityExpression as END/USE_INVOCATION_RESULT for HTTP/S." + " Priorities and gotoPriorityExpression will need to" + " be added modified/added manually in [{}].".format(str(cmd).strip())) + bind_cmd = '#' + str(cmd) + return bind_cmd + + def complete_convert_bind_cmd(self, cmd, policy_name, module, priority_arg, goto_arg, position): + # Pass bind command arguments to return converted bind commands + if (cmd.ot == "global"): + self.convert_global_bind( + cmd, cmd, policy_name, module, priority_arg, goto_arg, position) + if (cmd.ot == "vserver"): + self.convert_entity_policy_bind( + cmd, cmd, policy_name, module, priority_arg, goto_arg, position) diff --git a/nspepi/nspepi2/convert_lb_cmd.py b/nspepi/nspepi2/convert_lb_cmd.py new file mode 100644 index 0000000..0267d1f --- /dev/null +++ b/nspepi/nspepi2/convert_lb_cmd.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python2 + +# Copyright 2021 Citrix Systems, Inc. All rights reserved. +# Use of this software is governed by the license terms, if any, +# which accompany or are included with this software. + +import logging +import copy +from collections import OrderedDict + +import nspepi_common as common +from nspepi_parse_tree import * +from convert_classic_expr import * +import convert_cli_commands as cli_cmds + +# All module names starting with "convert_" are parsed to detect and register +# class methods + + +@common.register_class_methods +class LB(cli_cmds.ConvertConfig): + """ + Handle lb vserver related commands. + """ + + def __init__(self): + """ + Information needed for conversion. + _search_patterns - compiled regular expression list + for searching CONTENT expression. + _match_patterns - compiled regular expression list + for matching CONTENT expression. + """ + content_patterns = [ + r'REQ\.HTTP\.URL\s+CONTENTS((\s+-length\s+\d+)?(\s+-offset\s+\d+)?)?', + r'URL\s+CONTENTS((\s+-length\s+\d+)?(\s+-offset\s+\d+)?)?', + r'REQ\.HTTP\.URLQUERY\s+CONTENTS((\s+-length\s+\d+)?(\s+-offset\s+\d+)?)?', + r'URLQUERY\s+CONTENTS((\s+-length\s+\d+)?(\s+-offset\s+\d+)?)?', + r'REQ\.HTTP\.HEADER\s+\S+\s+CONTENTS((\s+-length\s+\d+)?(\s+-offset\s+\d+)?)?' + ] + self._search_patterns = [] + self._match_patterns = [] + for pattern in content_patterns: + self._search_patterns.append(re.compile(pattern, re.I)) + self._match_patterns.append(re.compile(pattern + "$", re.I)) + + @common.register_for_cmd("add", "lb", "vserver") + def convert_lb_rule(self, add_lbvserver_parse_tree): + """ + Converts classic lb rule to advanced. + Syntax: + add lb vserver + -persistenceType -rule + to + add lb vserver + -persistenceType -rule + + In classic expressions only "CONTENTS" gives string as result. + All other classic expressions gives boolean as result. + In lb vserver, rule is used with -persistencetype rule. + 1. When the classic expression results to boolean(either + ns_true or ns_false), no persistencesessions are created. + If we convert the classic expression, then the + advanced expression will result in either true or false. + But for advanced expressions either true or false, + persistencesessions are created for true and false. + This will change the functionality. So to aviod this for + expressions which results in boolean, remove -rule and + -persistenceType. + + 2. When the classic expression is simple and contains CONTENTS, + replace with appropriate advanced expression. + Possible expressions with CONTENTS: + 1. REQ.HTTP.URL CONTENTS + 2. URL CONTENTS + 3. REQ.HTTP.URLQUERY CONTENTS + 4. URLQUERY CONTENTS + 5. REQ.HTTP.HEADER
CONTENTS + 6. RES.HTTP.HEADER
CONTENTS (Not handling response + expression because this is not valid for lb rule) + CONTENTS has length and offset option. + REQ.HTTP.URL CONTENTS -length -offset + + 3. When the classic expression is compound and contains CONTENTS, + error is thrown to convert the expression manually, because + &&, || operations are not supported on strings in advanced. + Example: + 1. "req.http.header hdr1 contents && req.http.header hdr2 contents" + 2. "req.http.header hdr1 contents && req.vlanid == 3 + || req.http.header hdr2 contents" + """ + lb_protocol = add_lbvserver_parse_tree.positional_value(1).value + lbv_name = add_lbvserver_parse_tree.positional_value(0).value + cli_cmds.vserver_protocol_dict[lbv_name] = lb_protocol.upper() + add_lbvserver_parse_tree = LB.convert_adv_expr_list( + add_lbvserver_parse_tree, ["Listenpolicy", "resRule", "pushLabel"]) + if not add_lbvserver_parse_tree.keyword_exists("rule"): + return [add_lbvserver_parse_tree] + + original_tree = copy.deepcopy(add_lbvserver_parse_tree) + rule = add_lbvserver_parse_tree.keyword_value("rule")[0].value + suffix_len_to_remove = len('.LENGTH.GT(0)"') + for index in range(len(self._search_patterns)): + if self.search_pattern(rule, index): + match_obj = self.match_pattern(rule, index) + if match_obj[0]: + """ CONTENTS exists and is a simple expression. + Old nspepi tool with -e is used to convert the expression. + Tool converts CONTENT expressions in following way: + "REQ.HTTP.URL CONTENTS" + to + "HTTP.REQ.URL.LENGTH.GT(0)" + appends ".LENGTH.GT(0)" to get result as boolean when + expression is used in policies. + But when used in lb vserver, .LENGTH.GT(0) should not + be added. + """ + rule = match_obj[1] + converted_rule = convert_classic_expr(rule) + # Removing ".length.get(0)" + converted_rule = converted_rule[:-suffix_len_to_remove] + \ + "\"" + add_lbvserver_parse_tree.keyword_value("rule")[0]. \ + set_value(converted_rule, True) + add_lbvserver_parse_tree.set_upgraded() + else: + # CONTENTS exists but not a simple expression. + # Throw error and don't convert. + logging.error(("-rule in the following command has to be " + "converted manually: {}").format( + str(add_lbvserver_parse_tree).strip())) + return [add_lbvserver_parse_tree] + + # Case when there is no CONTENT in expression. + add_lbvserver_parse_tree = LB.convert_keyword_expr( + add_lbvserver_parse_tree, "rule") + if add_lbvserver_parse_tree.upgraded: + removed_keywords = [] + persistencetypes = ["rule", "urlpassive", "customserverid"] + """ + When rule results in boolean value, persistenceType or lbMethod + should be removed in the following cases. + 1. If persistenceType value is rule and resRule keyword exists, + then persistenceType keyword should not be removed. + 2. If persistenceType value is rule and resRule keyword does not + exists, then persistenceType keyword should be removed. + 2. If persistenceType value is urlpassive or customserverid, + then persistenceType should be removed. + 3. If lbMethod is Token, then lbMethod should be removed. + """ + if (add_lbvserver_parse_tree.keyword_exists("persistenceType") and + add_lbvserver_parse_tree.keyword_value("persistenceType")[0]. + value.lower() in persistencetypes and not + add_lbvserver_parse_tree.keyword_exists("resRule")): + add_lbvserver_parse_tree.remove_keyword("persistenceType") + removed_keywords.append("persistenceType") + if (add_lbvserver_parse_tree.keyword_exists("lbMethod") and + add_lbvserver_parse_tree.keyword_value("lbMethod")[0].value. + lower() == "token"): + add_lbvserver_parse_tree.remove_keyword("lbMethod") + removed_keywords.append("lbMethod") + if len(removed_keywords) > 0: + removed_keywords.append("rule") + add_lbvserver_parse_tree.remove_keyword("rule") + logging.warning(("-rule classic expression results in boolean " + "value. The equivalent advanced expression " + "will result boolean value in string " + "format. This will result in functionality " + "change when rule is used for persistenceType" + " or lbMethod. To aviod the functionality " + "change, {} command is modified by removing " + "the following keywords: {}.").format( + str(original_tree).strip(), + ", ".join(removed_keywords))) + return [add_lbvserver_parse_tree] + + def search_pattern(self, rule, index): + """ + Searches for CONTENT expression in rule expression and in + named expressions if included. + Returns True if CONTENT expression is found. + rule - Expression in which CONTENT expression should be searched. + index - CONTENT expression index in _search_patterns list. + """ + expr_list = cli_cmds.get_classic_expr_list(rule) + if self._search_patterns[index].search(rule): + return True + else: + for expr in expr_list: + expr_name = expr[0] + expr_rule = cli_cmds.named_expr[expr_name] + if self.search_pattern(expr_rule, index): + return True + return False + + def match_pattern(self, rule, index): + """ + Matches for CONTENT expression in rule expression and in + named expressions if included. + rule - Expression which should be matched with CONTENT expression. + index - CONTENT expression index in _match_patterns list. + Returns 2 values: + boolean value - True if macthed with CONTENT expression. + rule - Rule with which expression got matched. + """ + expr_list = cli_cmds.get_classic_expr_list(rule) + if self._match_patterns[index].match(rule): + return [True, rule] + else: + if len(expr_list) == 0 or len(expr_list) > 1: + return [False] + else: + # Case when one named expression exists in rule. + expr_name = expr_list[0][0] + # When rule is complex. + # Example: "true && e1" + if rule != expr_name: + return [False] + expr_rule = cli_cmds.named_expr[expr_name] + return self.match_pattern(expr_rule, index) + + @common.register_for_cmd("bind", "lb", "vserver") + def convert_lb_vserver_bind(self, bind_parse_tree): + """ + Handles lb vserver bind command. + bind lb vserver -policyName + """ + if not bind_parse_tree.keyword_exists('policyName'): + return [bind_parse_tree] + + policy_name = bind_parse_tree.keyword_value("policyName")[0].value + policy_type = common.pols_binds.get_policy(policy_name).module + priority_arg = "priority" + goto_arg = "gotoPriorityExpression" + + """ + Calls the method that is registered for the particular + policy type that is bound to LB. Returns converted_list. + If the policy module is not registered for binding, + then returns the original parse tree. + """ + key = "LB" + if key in common.bind_table: + if policy_type in common.bind_table[key]: + m = common.bind_table[key][policy_type] + return m.method(m.obj, bind_parse_tree, policy_name, + priority_arg, goto_arg) + return [bind_parse_tree] diff --git a/nspepi/nspepi2/convert_patclass_commands.py b/nspepi/nspepi2/convert_patclass_commands.py new file mode 100644 index 0000000..e0ce52a --- /dev/null +++ b/nspepi/nspepi2/convert_patclass_commands.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python2 + +# Copyright 2021 Citrix Systems, Inc. All rights reserved. +# Use of this software is governed by the license terms, if any, +# which accompany or are included with this software. + +import nspepi_common as common +import convert_cli_commands as cli_cmds +from nspepi_parse_tree import * + +# All module names starting with "convert_" are parsed to detect and register +# class methods + +@common.register_class_methods +class PATCLASS(cli_cmds.ConvertConfig): + + @common.register_for_cmd("add", "policy", "patclass") + def convert_add_patclass(self, tree): + """ + Process: add policy patclass + + Args: + tree: Command parse tree for add policy patclass command + + Returns: + tree: Processed command parse tree for add policy patclass command + """ + patset_tree = CLICommand('add', 'policy', 'patset') + name = CLIPositionalParameter(tree.positional_value(0).value) + patset_tree.add_positional(name) + return [patset_tree] + + @common.register_for_cmd("bind", "policy", "patclass") + def convert_bind_patclass(self, tree): + """ + Process: bind policy patclass + + Args: + tree: Command parse tree for bind policy patclass command + + Returns: + tree: Processed command parse tree for bind policy patclass command + """ + patset_tree = CLICommand('bind', 'policy', 'patset') + name = CLIPositionalParameter(tree.positional_value(0).value) + pattern = CLIPositionalParameter(tree.positional_value(1).value) + patset_tree.add_positional(name) + patset_tree.add_positional(pattern) + return [patset_tree] diff --git a/nspepi/nspepi2/convert_responder_command.py b/nspepi/nspepi2/convert_responder_command.py new file mode 100644 index 0000000..d44a53f --- /dev/null +++ b/nspepi/nspepi2/convert_responder_command.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python2 + +# Copyright 2021 Citrix Systems, Inc. All rights reserved. +# Use of this software is governed by the license terms, if any, +# which accompany or are included with this software. + +from nspepi_parse_tree import * +import convert_cli_commands as cli_cmds + +@common.register_class_methods +class Responder(cli_cmds.ConvertConfig): + """ + Handles responder feature to store few information for + filter bind conversion + resp_global_goto_exists - Set this true if existing responder policy is + globally bound with GOTO END/USE_INVOCATION_RESULT + resp_vserver_goto_exists - Set this true if existing responder policy is + bound to vserver with GOTO END/USE_INVOCATION_RESULT + """ + resp_global_goto_exists = False + resp_vserver_goto_exists = False + + def __init__(self): + self._noop_action_list = [] + + @common.register_for_cmd("add", "responder", "action") + def convert_responder_action(self, tree): + """ + add responder action + """ + action_name = tree.positional_value(0).value.lower() + action_type = tree.positional_value(1).value.lower() + # If action is noop, then don't return action + if (action_type == "noop"): + self._noop_action_list.append(action_name) + return [] + tree = Responder.convert_adv_expr_list( + tree, [2, "reasonPhrase", "headers"]) + return [tree] + + @common.register_for_cmd("add", "responder", "policy") + def convert_responder_policy(self, tree): + """ + Saved policy name in policy_list. + add responder policy + """ + policy_name = tree.positional_value(0).value + policy_action = tree.positional_value(2).value.lower() + if (policy_action in self._noop_action_list): + tree.positional_value(2).set_value("NOOP") + tree.set_upgraded() + + pol_obj = common.Policy(policy_name, self.__class__.__name__, + "advanced") + common.pols_binds.store_policy(pol_obj) + tree = Responder.convert_adv_expr_list(tree, [1]) + return [tree] + + @common.register_for_cmd("bind", "responder", "global") + def convert_responder_global(self, tree): + """ + Handles responder global bind command. + bind responder global + [] [-type ] + When responder policy is bound: + 1. Check if GOTO is END/USE_INVOCATION_RESULT for + HTTP/SSL vservers + tree - bind command parse tree + """ + get_goto_arg = tree.positional_value(2).value + policy_name = tree.positional_value(0).value + get_bind_type = tree.keyword_value("type")[0].value + module = self.__class__.__name__ + priority_arg = 1 + goto_arg = 2 + position = "inplace" + if get_bind_type in ("REQ_OVERRIDE", "REQ_DEFAULT"): + # Set below flags only if added vserver is of HTTP/SSL protocol + if get_goto_arg.upper() in ("END", "USE_INVOCATION_RESULT"): + Responder.resp_global_goto_exists = True + self.convert_global_bind( + tree, tree, policy_name, module, priority_arg, goto_arg, position) + return [] + + @common.register_for_bind(["LB", "ContentSwitching", "CacheRedirection"]) + def convert_responder_vserver_bindings( + self, bind_parse_tree, policy_name, priority_arg, goto_arg): + """ + Handles responder policy bindings to vservers - LB, CS, CR + Syntax for responder policy binding: + bind lb/cr/cs vserver -policyName + -priority -gotoPriorityExpression + -type REQUEST + When responder policy is bound: + 1. Check if GOTO is END/USE_INVOCATION_RESULT for HTTP/SSL vservers + """ + get_goto_arg = bind_parse_tree.keyword_value( + "gotoPriorityExpression")[0].value + policy_name = bind_parse_tree.keyword_value("policyName")[0].value + vs_name = bind_parse_tree.positional_value(0).value + module = self.__class__.__name__ + priority_arg = "priority" + goto_arg = "gotoPriorityExpression" + if cli_cmds.vserver_protocol_dict[vs_name] in ("HTTP", "SSL"): + # Set below flags only if vserver is of ptotocol HTTP/SSL + if get_goto_arg.upper() in ("END", "USE_INVOCATION_RESULT"): + Responder.resp_vserver_goto_exists = True + self.convert_entity_policy_bind( + bind_parse_tree, bind_parse_tree, policy_name, + module, priority_arg, goto_arg) + return [] + diff --git a/nspepi/nspepi2/convert_rewrite_command.py b/nspepi/nspepi2/convert_rewrite_command.py new file mode 100644 index 0000000..cdf7ec4 --- /dev/null +++ b/nspepi/nspepi2/convert_rewrite_command.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python2 + +# Copyright 2021 Citrix Systems, Inc. All rights reserved. +# Use of this software is governed by the license terms, if any, +# which accompany or are included with this software. + +from nspepi_parse_tree import * +import convert_cli_commands as cli_cmds +from cli_lex import * + +@common.register_class_methods +class Rewrite(cli_cmds.ConvertConfig): + """ + Handles Rewrite feature to store few information for + filter bind conversion + rw_global_goto_exists - Set this true if existing rewrite policy is + globally bound with GOTO END/USE_INVOCATION_RESULT + rw_vserver_goto_exists - Set this true if existing rewrite policy is bound + to vserver with GOTO END/USE_INVOCATION_RESULT + """ + rw_global_goto_exists = False + rw_vserver_goto_exists = False + + @staticmethod + def is_pattern_regex(pattern): + """ + Helper function to check whether the + given pattern is a regex pattern or + a text string. + """ + pat_len = len(pattern) + if (pat_len < 5): + return (False) + delimiter = pattern[2] + r_pat = pattern[0].lower() + e_pat = pattern[1].lower() + if ((r_pat != 'r') or (e_pat != 'e') or + (pattern[pat_len - 1] != delimiter) or + Lexer.adv_ident_char(delimiter)): + return (False) + return (True) + + @common.register_for_cmd("add", "rewrite", "action") + def convert_rewrite_action(self, tree): + if tree.keyword_exists('pattern'): + pattern_value = tree.keyword_value('pattern')[0].value + tree.remove_keyword('pattern') + search_key = CLIKeywordName('search') + if Rewrite.is_pattern_regex(pattern_value): + search_val = "regex(" + pattern_value + ")" + else: + tree_obj = CLIParseTreeNode() + pattern_value = tree_obj.normalize(pattern_value, True) + search_val = "text(" + pattern_value + ")" + search_val_param = CLIKeywordParameter(search_key) + search_val_param.add_value(search_val) + tree.add_keyword(search_val_param) + tree = Rewrite.convert_adv_expr_list(tree, [3, "refineSearch"]) + return [tree] + + @common.register_for_cmd("add", "rewrite", "policy") + def convert_rewrite_policy(self, tree): + """ + Saved policy name in policy_list. + add rewrite policy + """ + policy_name = tree.positional_value(0).value + pol_obj = common.Policy(policy_name, self.__class__.__name__, + "advanced") + common.pols_binds.store_policy(pol_obj) + tree = Rewrite.convert_adv_expr_list(tree, [1]) + return [tree] + + @common.register_for_cmd("bind", "rewrite", "global") + def convert_rewrite_global(self, tree): + """ + Handles rewrite global bind command. + bind rewrite global + [] [-type ] + Store GOTO info if GOTO is END/USE_INVOCATION_RESULT for + HTTP/SSL vservers + tree - bind command parse tree + """ + get_goto_arg = tree.positional_value(2).value + policy_name = tree.positional_value(0).value + get_bind_type = tree.keyword_value("type")[0].value + module = self.__class__.__name__ + priority_arg = 1 + goto_arg = 2 + if get_bind_type in ( + "REQ_OVERRIDE", "RES_OVERRIDE", "REQ_DEFAULT", "RES_DEFAULT"): + # Set below flags only if added vserver is of HTTP/SSL protocol + if get_goto_arg.upper() in ("END", "USE_INVOCATION_RESULT"): + Rewrite.rw_global_goto_exists = True + self.convert_global_bind( + tree, tree, policy_name, module, priority_arg, goto_arg) + return [] + + @common.register_for_bind(["LB", "ContentSwitching", "CacheRedirection"]) + def convert_rewrite_vserver_bindings( + self, bind_parse_tree, policy_name, priority_arg, goto_arg): + """ + Handles rewrite policy bindings to vservers - LB, CS, CR + Syntax for rewrite policy binding: + bind lb/cr/cs vserver -policyName + -priority -gotoPriorityExpression + -type [REQUEST | RESPONSE] + When rewrite policy is bound: + 1. Store GOTO info if GOTO is END/USE_INVOCATION_RESULT for + HTTP/SSL vservers + 2. vserver_protocol_dict - dict from cli_cmds and convert_lb_cmd + packages which carries protocol as value to the key - vserver name + """ + get_goto_arg = bind_parse_tree.keyword_value( + "gotoPriorityExpression")[0].value + vs_name = bind_parse_tree.positional_value(0).value + policy_name = bind_parse_tree.keyword_value("policyName")[0].value + module = self.__class__.__name__ + priority_arg = "priority" + goto_arg = "gotoPriorityExpression" + if cli_cmds.vserver_protocol_dict[vs_name] in ("HTTP", "SSL"): + # Set below flags only if vserver binding policies is of + # HTTP/SSL protocol + if get_goto_arg.upper() in ("END", "USE_INVOCATION_RESULT"): + Rewrite.rw_vserver_goto_exists = True + self.convert_entity_policy_bind( + bind_parse_tree, bind_parse_tree, policy_name, + module, priority_arg, goto_arg) + return [] + diff --git a/nspepi/nspepi2/nspepi_common.py b/nspepi/nspepi2/nspepi_common.py new file mode 100644 index 0000000..9dd6781 --- /dev/null +++ b/nspepi/nspepi2/nspepi_common.py @@ -0,0 +1,1056 @@ +#!/usr/bin/env python2 + +# Copyright 2021 Citrix Systems, Inc. All rights reserved. +# Use of this software is governed by the license terms, if any, +# which accompany or are included with this software. + +""" +Common data for nspepi accessible from other modules. +Dependency packages: None +""" + +import collections +import copy +import functools +import itertools +import logging +import sys +import os +import inspect + + +import nspepi_parse_tree + +currentfile = os.path.abspath(inspect.getfile(inspect.currentframe())) +currentdir = os.path.dirname(currentfile) +parentdir = os.path.dirname(currentdir) +sys.path.insert(0, parentdir) + + +def get_nspepi_tool_path(): + """ Get the path of old nspepi tool. + This function will check whether the tool + exists in the current directory or the + parent directory, and return the path + based on that.""" + file_found = False + filename = 'nspepi_helper' + for path in [currentdir, parentdir]: + candidate = os.path.join(path, filename) + if os.path.isfile(candidate): + file_found = True + break + if file_found: + return os.path.abspath(candidate) + else: + return None + + +CMD_MOD_ERR_MSG = (" Advanced expressions only have a fixed ordering of the" + " types of bindings without interleaving, except that" + " global bindings are allowed before all other bindings" + " and after all bindings. If you have global bindings" + " in the middle of non-global bindings or any other" + " interleaving then you will need to reorder all your" + " bindings for that feature and direction." + " Refer to nspepi documentation.") +# store registered methods to be called for parsed CLI commands +dispatchtable = collections.defaultdict(list) +# store registered methods to be called at end of processing CLI commands +final_methods = [] +# store registered methods to be called before the start of processing of +# CLI commands. +init_methods = [] +# store registered methods to be called for vserver/service/user/group +# bindings. +# Two level dictionary: bind_table[bind_cmd_module][policy_type] +bind_table = collections.OrderedDict() + + +class DispatchData(object): + """ + Store object and method to dispatch call to for parsed CLI command. + """ + def __init__(self, o, m): + self.obj = o + self.method = m + + +def register_class_methods(cls): + """ + Decorator that registers the tagged methods within the class. + NOTE: Class MUST BE decorated with this if it has methods decorated by + by decorators in this module otherwise they won't be registered! + Also, decorate base class with this as well if it has methods + decorated by decorators in this module. + + Args: + cls: The class to be processed for tagged methods. + + Returns: + cls: The class itself that was passed in as an argument. + """ + obj = cls() + class_name = obj.__class__.__name__ + for name, method in cls.__dict__.items(): + if hasattr(method, "register_for_cmd"): + for cmd in getattr(method, "cmd_list"): + key = " ".join([cmd['op'], cmd['group'], cmd['ot']]).lower() + dispatchtable[key].append(DispatchData(obj, method)) + if hasattr(method, "register_for_final_call"): + final_methods.append(DispatchData(obj, method)) + if hasattr(method, "register_for_bind"): + for bind_module in getattr(method, "module_list"): + if bind_module not in bind_table: + bind_table[bind_module] = collections.OrderedDict() + bind_table[bind_module][class_name] = DispatchData(obj, method) + if hasattr(method, "register_for_init_call"): + init_methods.append(DispatchData(obj, method)) + return cls + + +def register_for_cmd(op, group, ot): + """ + Decorator that tags a method to be registered to process a command. + NOTE: This decorator MUST BE the outermost decorator used on a method + decorated with multiple decorators! The custom user attributes + of a method aren't preserved in python 2.7 even with use of + wraps() from functools. + + Args: + op: Operation of command such as "add", "bind", etc. + group: Group of command such as "lb", "responder", etc. + ot: Object type of command such as "vserver", "policy", etc. + + Returns: + m: The same method itself as passed in the arguments but with + additional user attributes set within the method. These + custom attributes are processed afterwards within the class + decorator. Once all of the decorators on the methods within + the class are executed and all of the methods and data in + the class is parsed then the class is formed and available. + At that time the class decorator gets called on the class + where these tagged methods by decorators like these are + processed. + """ + def wrapper(m): + if not hasattr(m, "register_for_cmd"): + m.cmd_list = [] + m.register_for_cmd = True + cmd = {'op': op, 'group': group, 'ot': ot} + m.cmd_list.append(cmd) + return m + return wrapper + + +def register_for_final_call(m): + """ + Decorator that tags a method to be called at the end of processing cmds. + NOTE: This decorator MUST BE the outermost decorator used on a method + decorated with multiple decorators! The custom user attributes + of a method aren't preserved in python 2.7 even with use of + wraps() from functools. + + Args: + m: Method to be tagged to call at end of processing + + Returns: + m: The same method itself as passed in the arguments but with + additional user attributes set within the method. These + custom attributes are processed afterwards within the class + decorator. Once all of the decorators on the methods within + the class are executed and all of the methods and data in + the class is parsed then the class is formed and available. + At that time the class decorator gets called on the class + where these tagged methods by decorators like these are + processed. + """ + m.register_for_final_call = True + return m + + +def register_for_init_call(m): + """ + Decorator that tags a method to be called at the end of processing cmds. + NOTE: This decorator MUST BE the outermost decorator used on a method + decorated with multiple decorators! The custom user attributes + of a method aren't preserved in python 2.7 even with use of + wraps() from functools. + + Args: + m: Method to be tagged to call at the start of processing + + Returns: + m: The same method itself as passed in the arguments but with + additional user attributes set within the method. These + custom attributes are processed afterwards within the class + decorator. Once all of the decorators on the methods within + the class are executed and all of the methods and data in + the class is parsed then the class is formed and available. + At that time the class decorator gets called on the class + where these tagged methods by decorators like these are + processed. + """ + m.register_for_init_call = True + return m + + +def register_for_bind(module_list): + """ + Decorator that tags a method to be registered to process a command. + NOTE: This decorator MUST BE the outermost decorator used on a method + decorated with multiple decorators! The custom user attributes + of a method aren't preserved in python 2.7 even with use of + wraps() from functools. + + Args: + module_list: List of all other module names where all present + module policy can be bound. + + Returns: + m: The same method itself as passed in the arguments but with + additional user attributes set within the method. These + custom attributes are processed afterwards within the class + decorator. Once all of the decorators on the methods within + the class are executed and all of the methods and data in + the class is parsed then the class is formed and available. + At that time the class decorator gets called on the class + where these tagged methods by decorators like these are + processed. + """ + def wrapper(m): + m.register_for_bind = True + m.module_list = module_list + return m + return wrapper + + +def dict_repr(obj): + """ + Creates an unambiguous and consistent representation of a dictionary. + + Args: + obj: The dictionary to produce the representation of + + Returns: + The string representation + """ + result = '{' + for key in sorted(obj): + elem = obj[key] + if isinstance(elem, dict): + result += repr(key) + ': ' + dict_repr(elem) + ', ' + else: + result += repr(key) + ': ' + repr(elem) + ', ' + if result.endswith(', '): + result = result[0:-2] + result += '}' + return result + + +def class_repr(obj): + """ + Creates an unambiguous and consistent representation of a class. + + Args: + obj: The class object to produce the representation of + + Returns: + The string representation + """ + return '<' + type(obj).__name__ + ' ' + dict_repr(obj.__dict__) + '>' + + +def get_cmd_arg(arg, cmd_tree): + """ + Return the indicated command argument from command parse tree. If arg is + a str it is a keyword. If arg is an int it is a positional index. If + the argument is not found, return None. + + Args: + arg: Command argument to look up + cmd_tree: Parsed command tree + + Returns: + value: Value of the command argument or None if not found + """ + value = None + if isinstance(arg, int): + node = cmd_tree.positional_value(arg) + if node is not None: + value = node.value + elif isinstance(arg, str): + if cmd_tree.keyword_exists(arg): + value = cmd_tree.keyword_value(arg)[0].value + else: + assert False, ("get_cmd_arg(arg, cmd_tree): arg {} not an instance of" + " int or str".format(arg)) + return value + + +class Group(object): + """ + Represents a group command based on a parsed command and provides + methods to read and set info based on analysis. + """ + def __init__(self, name="", weight="0"): + """ + Construct object based on parsed group command parameters. + + Args: + name: Name of the group + weight: Weight of the group + """ + self.name = name + self.weight = weight + + def __repr__(self): + """ Creates an unambiguous representation of the Group object. + + Returns: + str: the string representation + """ + return class_repr(self) + + +class Policy(object): + """ + Represents a policy command based on a parsed command and provides + methods to read and set info based on analysis. + """ + def __init__(self, name="", module="", policy_type=""): + """ + Construct object based on parsed policy command parameters. + + Args: + name: Name of the policy + module: Unambiguated name of the group of the policy + (ex. responder, TMSession for tm sessionPolicy, etc.) + policy_type: Type of the policy ("classic" or "advanced") + """ + self.name = name + self.module = module + self.policy_type = policy_type + + def __repr__(self): + """ Creates an unambiguous representation of the Policy object. + + Returns: + str: the string representation + """ + return class_repr(self) + + +class Bind(object): + """ + Represents a bind command based on a parsed command and provides + methods to read and set info based on analysis. + """ + def __init__(self, entity, entity_type, entity_name, policy_name, + policy_module, bind_type, priority, cmd_str, global_type="", + lineno="0"): + """ + Construct object based on parsed bind command parameters. + + Args: + entity: OT of the command (ex. vserver, user, group, service) + entity_type: Group of the command (ex. lb, cs, cr, ssl, aaa) + entity_name: Name of the entity (None if global) + policy_name: Name of the bound policy + policy_module: Module of the bound policy + bind_type: Type of the bind (ex. request, response, etc.) + priority: Priority of the bind command + cmd_str: String representation of the bind command + global_type: Suggested global type based on analysis + (override or default or "") + lineno: Line number of the original bind command in config file + """ + self.entity = entity + self.entity_type = entity_type + self.entity_name = entity_name + self.policy_name = policy_name + self.policy_module = policy_module + self.bind_type = bind_type + self.priority = priority + self.cmd_str = cmd_str + self.global_type = global_type + self.lineno = lineno + + def __repr__(self): + """ Creates an unambiguous representation of the Bind object. + + Returns: + str: the string representation + """ + return class_repr(self) + + +class PoliciesAndBinds(object): + """ + Holds policies and bind info for analysis. For ex. to detect if + a non-global priority has a lower number than a global priority + that in turn is a lower number priority than a priority for the + same non-global bind point. + """ + # Specify sort order below for use in priority analysis methods + # below to sort binds at the same priority in this order and + # also to determine whether the binds conform to this order + # to detect interleaving + ORDER = ["global", "user", "group", "vpn", "lb", "cs", "cr", "service", + "global"] + # List of the policy modules for which global override + # should be skipped. + skip_global_override = [] + # dictionary that holds groups + # key: name of group + # value: Group object + groups = collections.defaultdict(lambda: Group()) + # dictionary that holds policies + # key: name of policy + # value: Policy object + policies = collections.defaultdict(lambda: Policy()) + # dictionary that holds global bind commands for analysis + # global_binds[][] + # key: policy module of bound policy (ex. responder, appfw, etc.) + # value: another dictionary with: + # key: bind_type (ex. request/response) + # value: list of Bind objects representing bind command + global_binds = collections.defaultdict( # module + lambda: collections.defaultdict(list)) # bind_type + # dictionary that holds entity bind commands for analysis + # entity_binds[][][][] + # [] + # key: entity (ex. vserver, user, group, service, global) + # value: another dictionary with: + # key: entity_type (ex. lb, cs, cr, ssl, vpn, authorization, aaa) + # value: another dictionary with: + # key: entity_name (name of bind entity) + # value: another dictionary with: + # key: module (module of bound policy) + # value: another dictionary with: + # key: bind_type (ex. request/response) + # value: list of Bind objects + entity_binds = collections.defaultdict( # entity + lambda: collections.defaultdict( # entity_type + lambda: collections.defaultdict( # entity_name + lambda: collections.defaultdict( # module + lambda: collections.defaultdict(list))))) # bind_type + # dictionary that holds analysis results for bind commands + # key: str representation of original bind command + # value: another dictionary with: + # key: analysis result name for bind command + # value: analysis result value for bind command + # ex. key: "unsupported", value: True + priority_analysis_results = collections.defaultdict( # bind cmd + lambda: collections.defaultdict()) # dict of results + + def __init__(self): + pass + + @staticmethod + def add_to_skip_global_override(module_name): + """ + Adds policy module name to the list for which + global override should be skipped. + module_name - Policy module name + """ + PoliciesAndBinds.skip_global_override.append(module_name) + + @staticmethod + def get_skip_global_override(): + """ + Returns the list of modules for which global + override should be skipped. + """ + return PoliciesAndBinds.skip_global_override + + def store_group(self, groupobj): + """ + Store group object representing group command for analysis. + + Args: + groupobj: Group object representing group command + """ + PoliciesAndBinds.groups[groupobj.name] = groupobj + logging.debug("Stored group: {}".format(groupobj)) + + def get_group(self, groupname): + """ + Returns the group object for the passed in groupname. + + Args: + groupname: Name of the group to look up + """ + return PoliciesAndBinds.groups[groupname] + + def store_policy(self, policyobj): + """ + Store policy object representing policy command for analysis. + + Args: + policyobj: Policy object representing policy command + """ + PoliciesAndBinds.policies[policyobj.name] = policyobj + logging.debug("Stored policy: {}".format(policyobj)) + + def get_policy(self, policyname): + """ + Returns the policy object for the passed in policyname. + + Args: + policyname: Name of the policy to look up + """ + return PoliciesAndBinds.policies[policyname] + + def store_original_bind(self, bindobj): + """ + Store bind object representing bind command for analysis. + + Args: + bindobj: Bind object representing bind command + """ + if (bindobj.policy_name in PoliciesAndBinds.policies + and PoliciesAndBinds.policies[bindobj.policy_name].policy_type + == "classic"): + logging.debug("Stored bind: {}".format(bindobj)) + if bindobj.entity == "global": + (PoliciesAndBinds.global_binds + [bindobj.policy_module][bindobj.bind_type]).append(bindobj) + else: + (PoliciesAndBinds.entity_binds + [bindobj.entity][bindobj.entity_type][bindobj.entity_name] + [bindobj.policy_module][bindobj.bind_type]).append(bindobj) + + def do_priority_analysis(self, global_list, local_list, + skip_global_override=False): + """ + Do priority analysis and return list of binds that cannot be converted + for a particular module and bind_type. + + Args: + global_list: List of global binds per module and bindtype + local_list: List of local binds per entitytype and module + and bindtype + skip_global_override: Whether to skip global override + or not in analysis. + + Returns: + res: List of unsupported binds + res_gtypes: List of Bind objects with suggested 'global_type' + as "override" or "default" based on analysis + """ + res = [] + res_gtypes = [] + state = 1 if skip_global_override else 0 + states = ["global", "local", "global"] + g_types = ["override", "", "default"] + # store globals and locals combined in dictionary by priority + # key: priority + # value: list of bind object(s) + combined = collections.defaultdict(list) + [combined[int(o.priority)].append(o) for o in local_list + global_list] + # go through sorted combined list of priorities and determine if the + # bindings fall under globals (override) followed by locals and + # then by globals (default) based on priority + for prio in sorted(combined.keys()): + # there could be multiple binds at the same priority + # so go through each in order + for o in combined[prio]: + s = "global" if o.entity == "global" else "local" + # if type of prev priority or prev bind differs from curr then + # go to next type/state. + if s != states[state]: + if (state + 1) < len(states): + state += 1 + else: + # no more valid states so all priorities including + # and after current priority cannot be converted + res = list(itertools.chain(*(combined.values()))) + break + o.global_type = g_types[state] + res_gtypes.append(o) + logging.debug("do_priority_analysis(): ") + logging.debug("\nglobals: {}\n\nlocals: {}\n\nunsupported: {}" + "".format(global_list, local_list, res)) + return res, res_gtypes + + def analyze_vserver_priorities(self): + """ + Analyze priorities of all classic policies bound to global against + vserver entities. The goal is to detect if any classic policy bind + cannot be converted to global override, local, or global default + while preserving their original priorities. For those classic policy + binds that can't be converted their analysis results are stored in + priority_analysis_results. + """ + # store list of unsupported binds after analysis + unsupported = set() + # store list of bind objects with updated 'global_type' + updated_global_types = [] + # store list of merged binds for a particular entity_type + local_binds = collections.defaultdict( # entity_type (ex. lb or cs) + lambda: collections.defaultdict( # module + lambda: collections.defaultdict(list))) # bind_type + # iterate through all binds and merge all binds for a particular + # entity_type per module and bind_type in local_binds. + # i.e. combine entries for all VServers together, + # keeping all other dimensions separate. + ebinds = PoliciesAndBinds.entity_binds + entity = "vserver" + for entity_type in ebinds[entity]: + for vs in ebinds[entity][entity_type]: + for module in ebinds[entity][entity_type][vs]: + for bind_type in ebinds[entity][entity_type][vs][module]: + local_binds[entity_type][module][bind_type] += ( + ebinds[entity][entity_type][vs][module][bind_type]) + logging.debug("Local binds: {}".format(local_binds)) + # if a module only contains global binds or local binds then + # they can all be converted. However, if both global and local + # binds exist for a module then analyze them to determine if + # conversion is possible or not and store analysis results. + gbinds = PoliciesAndBinds.global_binds + for gmodule in gbinds: + for gbind_type in gbinds[gmodule]: + if local_binds: + for entity_type in local_binds: + locals_list = [] + if (gmodule in local_binds[entity_type] and gbind_type + in local_binds[entity_type][gmodule]): + # module and bind_type match for global and local + locals_list += ( + local_binds[entity_type][gmodule][gbind_type]) + logging.debug( + "do_priority_analysis() for {} {}" + "".format(gmodule, gbind_type)) + unsupp, updated_gtypes = self.do_priority_analysis( + gbinds[gmodule][gbind_type], locals_list, + gmodule in PoliciesAndBinds. + get_skip_global_override()) + unsupported.update(unsupp) + updated_global_types += updated_gtypes + else: + unsupp, updated_gtypes = self.do_priority_analysis( + gbinds[gmodule][gbind_type], [], + gmodule in PoliciesAndBinds. + get_skip_global_override()) + unsupported.update(unsupp) + updated_global_types += updated_gtypes + # store analysis results + res = PoliciesAndBinds.priority_analysis_results + for bindobj in unsupported: + res[bindobj.cmd_str]["unsupported"] = True + for bindobj in updated_global_types: + res[bindobj.cmd_str]['global_type'] = bindobj.global_type + + @staticmethod + def get_entity_state_name(o): + """ + Returns the computed name for the state of the passed-in bind object + for use in priority analysis. + + Args: + o - Bind object + """ + return o.entity if (o.entity == "global" or o.entity == "user" + or o.entity == "group") else o.entity_type + + @staticmethod + def entity_key(o): + """ + Return key for use in sorting all bind entities in order of + user, group, vpn, lb, cs, service and global at the same + priority for interleaving priority analysis. + + Args: + o - Bind object to determine key for sort + """ + # removing global from the beginning because at the same priority + # there's no concept of global override and global default. Based on + # investigation the globals always came after the locals so + # that's why global is only at the end here. + order = PoliciesAndBinds.ORDER[1:] + name = PoliciesAndBinds.get_entity_state_name(o) + return -1 if name not in order else order.index(name) + + def do_priority_analysis_for_all_entities( + self, global_list, local_list, skip_global_override=False): + """ + Do priority analysis and return list of binds that cannot be converted + for a single module and bindtype. + + Args: + global_list: List of global binds per module and bindtype + local_list: List of all local binds per module and bindtype + skip_global_override: Whether to skip global override + or not in analysis + + Returns: + res: List of unsupported binds + """ + res = [] + state = 1 if skip_global_override else 0 + # "global" is at the beginning of the list to account for global + # override. Based on investigation the globals always + # came after the locals so that's why globals are put at the end + # in "ORDER". In case of priority 0 with multiple local binds as + # well at priority 0 all of the globals can be converted to be + # global default. The reason for global being at the front here + # is because if there are only global binds at a single priority + # then they can be converted to be global override. More importantly + # if there are globals of low numbered priority then all locals after + # that in priority and then globals after that in priority we can + # also support that using a combination of global override and + # global default. + states = PoliciesAndBinds.ORDER + # store globals and locals combined in dictionary by priority + # key: priority + # value: list of bind object(s) + combined = collections.defaultdict(list) + [combined[int(o.priority)].append(o) for o in local_list + global_list] + # go through sorted combined list of priorities and determine if the + # bindings fall under the order specified above in variable "states" + for prio in sorted(combined.keys()): + # there could be multiple binds at the same priority + # so go through each in sorted order based on order + # specified above in variable "states" + for o in sorted(combined[prio], key=PoliciesAndBinds.entity_key): + s = PoliciesAndBinds.get_entity_state_name(o) + # if entity of prev priority or prev bind differs from curr + # then go to next type/state + if s != states[state]: + # if entity's group/"state" is not applicable then skip + # for cr, gslb and authentication. Evaluation for them + # only involve globals and don't support combination + # with others. + if s not in states: + continue + # if curr bind is of a prev entity group/"state" that + # has already been processed then mark as unsupported. + # Obtain index of current state from states but for + # global pick the first ("override") and last ("default") + # correctly based on whether current state is past + # global "override" by passing in the start index + # as 0 if current state is at global "override" or 1 + # if current state is past global "override" to pick + # global "default" for current state of "global". + state_index = states.index(s, 0 if state == 0 else 1) + if state_index < state: + logging.debug("state {} for {} is already processed" + " as current state is at {} so marking" + " as unsupported" + "".format(s, o.cmd_str, states[state])) + res.append(o) + continue + # go to next state if possible and check if curr bind + # falls under that + while s != states[state]: + if (state + 1) < len(states): + state += 1 + else: + # no more valid states so all binds at current + # priority and after cannot be converted + logging.debug("no more valid states so marking" + " as unsupported for {}" + "".format(o.cmd_str)) + res.append(o) + break + logging.debug("do_priority_analysis_for_all_entities(): ") + logging.debug("\nglobals: {}\n\nlocals: {}\n\nunsupported: {}" + "".format(global_list, local_list, res)) + return res + + def analyze_multiple_entities_for_interleaving_priorities(self): + """ + Analyze priorities of all classic policies bound to global against + all entities for interleaving priorities. The goal is to detect if + any classic policy bind cannot be converted to global override, + user, group, vpn, lb, cs, service, global default in that order + while preserving their original priorities. For those classic policy + binds that can't be converted their analysis results are stored in + priority_analysis_results. + """ + # store list of unsupported binds after analysis + unsupported = set() + # store list of merged binds for all entity_types + local_binds = collections.defaultdict( # module of policy + lambda: collections.defaultdict(list)) # bind_type + # iterate through all binds and merge all binds for each module and + # bind_type in local_binds. + # i.e. combine entries for all VServers and types of entities + # together, keeping all other dimensions separate. + ebinds = PoliciesAndBinds.entity_binds + for entity in ebinds: + for entity_type in ebinds[entity]: + for vs in ebinds[entity][entity_type]: + for module in ebinds[entity][entity_type][vs]: + for bind_type in (ebinds + [entity][entity_type][vs][module]): + local_binds[module][bind_type] += ( + ebinds + [entity][entity_type][vs][module][bind_type]) + logging.debug("analyze_multiple_entities_for_interleaving_priorities:") + logging.debug("Local binds: {}".format(local_binds)) + # if a module contains only global binds then they can all be + # converted. However, if only local binds or both global and local + # binds exist for a module then analyze them to determine if + # conversion is possible or not and store analysis results. + gbinds = PoliciesAndBinds.global_binds + for module in local_binds: + for bind_type in local_binds[module]: + globals_list = [] + if module in gbinds and bind_type in gbinds[module]: + # module and bind_type match for both global and local + globals_list += gbinds[module][bind_type] + logging.debug( + "self.do_priority_analysis_for_all_entities() for {} {}" + "".format(module, bind_type)) + unsupported.update( + self.do_priority_analysis_for_all_entities( + globals_list, local_binds[module][bind_type], + module in PoliciesAndBinds. + get_skip_global_override())) + # store analysis results + res = PoliciesAndBinds.priority_analysis_results + for bindobj in unsupported: + res[bindobj.cmd_str]["unsupported"] = True + + def do_priority_analysis_for_all_users_groups(self, user_list, group_list): + """ + Do priority analysis for all users and groups and return list of binds + that cannot be converted for a single module and bindtype. + Check for and handle the following cases: + 1. Mark bind as unsupported for any user bind that comes after a group + bind in priority + 2. Mark bind as unsupported for any earlier group bind that comes + after a different group bind in priority. This is to detect any + interleaving between groups. + 3. Mark bind as unsupported for any group bind that conflicts with + the weight of the group. In advanced policy evaluation groups + are evaluated in order of their weight so detect if any classic + binds are at priorities contradictory to the groups' weights. + 4. Check if any groups have the same weight and give an error + + Args: + user_list: List of user binds per module and bindtype + group_list: List of group binds per module and bindtype + + Returns: + res: List of unsupported binds + """ + res = set() + state = 0 + states = ["user", "group"] + # store users and groups combined in dictionary by priority + # key: priority + # value: list of bind object(s) + combined = collections.defaultdict(list) + [combined[int(o.priority)].append(o) for o in user_list + group_list] + # check for case #1 described in method comments above + # go through sorted combined list of priorities and determine if the + # bindings fall under the order specified above in variable "states" + for prio in sorted(combined.keys()): + # there could be multiple binds at the same priority + # so go through each in sorted order based on order + # specified above in variable "states" + for o in sorted(combined[prio], + key=lambda b: states.index(b.entity)): + s = o.entity + # if entity of prev priority or prev bind differs from curr + # then go to next type/state + if s != states[state]: + # if curr bind is of a prev "state" that has already been + # processed then mark as unsupported. This is to handle + # case #1 described in method comments above. + if states.index(s) < state: + logging.debug("state {} for {} is already processed" + " as current state is at {} so marking" + " as unsupported" + "".format(s, o.cmd_str, states[state])) + res.add(o) + continue + # go to next state if possible and check if curr bind + # falls under that + while s != states[state]: + if (state + 1) < len(states): + state += 1 + else: + # no more valid states so all binds at current + # priority and after cannot be converted + logging.debug("no more valid states so marking" + " as unsupported for {}" + "".format(o.cmd_str)) + res.add(o) + break + # check for case #2 described in method comments above + # store groups in dictionary by priority + # key: priority + # value: list of group bind object(s) + groups = collections.defaultdict(list) + [groups[int(o.priority)].append(o) for o in group_list] + # store list of groups that have already been processed + # at earlier priorities + earlier = [""] + # go through sorted group list of priorities and determine if + # any earlier group bind comes after a different group bind + # indicating interleaving between groups + for prio in sorted(groups.keys()): + # there could be multiple binds at the same priority + # so go through each in sorted order based on ns.conf order + # and check for interleaving. + for o in sorted(groups[prio], key=lambda o: int(o.lineno)): + if o.entity_name in earlier and o.entity_name != earlier[-1]: + logging.debug("group {} for {} is already processed" + " earlier so marking as unsupported" + "".format(o.entity_name, o.cmd_str)) + res.add(o) + continue + elif o.entity_name != earlier[-1]: + earlier.append(o.entity_name) + # store max weight seen for groups processed + max_weight = 0 + # check for case #3 described in method comments above + # go through sorted group list of priorities and determine + # if the weights of the group are in contradictory order + # to the priorities + for prio in sorted(groups.keys()): + # there could be multiple binds at the same priority + # so go through each and check for interleaving. + for o in sorted(groups[prio], key=lambda o: int(o.lineno)): + # lower numbered weights indicate higher preference of the + # group compared to higher numbered weights + w = int(self.get_group(o.entity_name).weight) + # at increasing priorities if there's a group encountered + # whose weight is less than the largest group weight seen + # of groups at earlier priorities then mark it as + # unsupported + if w < max_weight: + logging.debug("group {} for {} has weight {} less than" + " max weight {} for an earlier group so" + " marking as unsupported" + "".format( + o.entity_name, o.cmd_str, w, max_weight)) + res.add(o) + continue + elif w > max_weight: + max_weight = w + # check for case #4 described in method comments above + # go through sorted group list in order weights and give + # an error if more than one group has the same weight + weights = collections.defaultdict(set) + [weights[int(self.get_group(o.entity_name).weight)].add(o) + for o in group_list] + for v in weights.itervalues(): + if len(set([o.entity_name for o in v])) > 1: + logging.error("Groups: {} having the same weight and bindings" + " have no defined ordering in Advanced Policy" + " evaluation.".format( + ", ".join(set([o.entity_name for o in v])))) + res.update(v) + logging.debug("do_priority_analysis_for_all_users_groups(): ") + logging.debug("\nusers: {}\n\ngroups: {}\n\nunsupported: {}" + "".format(user_list, group_list, res)) + return res + + def analyze_user_group_priorities(self): + """ + Analyze user and group priorities. + """ + # store list of unsupported binds after analysis + unsupported = set() + # store list of merged binds for all users + user_binds = collections.defaultdict( # module of policy + lambda: collections.defaultdict(list)) # bind_type + # store list of merged binds for all groups + group_binds = collections.defaultdict( # module of policy + lambda: collections.defaultdict(list)) # bind_type + # iterate through all binds and merge all binds for each module and + # bind_type in user_binds and group_binds. + # i.e. combine entries for all users and groups + # respectively, keeping all other dimensions separate. + ebinds = PoliciesAndBinds.entity_binds + for entity in ["user", "group"]: + for entity_type in ebinds[entity]: + for name in ebinds[entity][entity_type]: + for module in ebinds[entity][entity_type][name]: + for bind_type in (ebinds + [entity][entity_type][name][module]): + if entity == "user": + user_binds[module][bind_type] += ( + ebinds[entity][entity_type][name][module] + [bind_type]) + elif entity == "group": + group_binds[module][bind_type] += ( + ebinds[entity][entity_type][name][module] + [bind_type]) + logging.debug("analyze_user_group_priorities():") + logging.debug("user binds: {}".format(user_binds)) + logging.debug("group binds: {}".format(group_binds)) + # if a module contains only user binds then they can all be + # converted. However, if only group binds or both user and group + # binds exist for a module then analyze them to determine if + # conversion is possible or not and store analysis results. + for module in group_binds: + for bind_type in group_binds[module]: + user_list = [] + if module in user_binds and bind_type in user_binds[module]: + # module and bind_type match for both group and user + user_list += user_binds[module][bind_type] + logging.debug( + "do_priority_analysis_for_all_users_groups() for {} {}" + "".format(module, bind_type)) + unsupported.update( + self.do_priority_analysis_for_all_users_groups( + user_list, group_binds[module][bind_type])) + # store analysis results + res = PoliciesAndBinds.priority_analysis_results + for bindobj in unsupported: + res[bindobj.cmd_str]["unsupported"] = True + + def analyze(self): + """ + Run analysis methods on PoliciesAndBinds. + """ + self.analyze_vserver_priorities() + self.analyze_user_group_priorities() + self.analyze_multiple_entities_for_interleaving_priorities() + + def is_bind_unsupported(self, orig_bind_str): + """ + Determine if bind is unsupported for passed in bind command string. + + Args: + orig_bind_str: Bind command string of original bind command read + from config + + Returns: + result: Priority analysis result for passed in bind command or + None if no result is present for it + """ + result = None + res = PoliciesAndBinds.priority_analysis_results + if orig_bind_str in res and "unsupported" in res[orig_bind_str]: + result = res[orig_bind_str]["unsupported"] + return result + + def get_global_type_for_bind(self, orig_bind_str): + """ + Return the global type based on analysis for passed in bind command + string. + + Args: + orig_bind_str: Bind command string of original bind command read + from config + + Returns: + result: Global type based on analysis result for passed in bind + command or None if no result is present for it + """ + result = None + res = PoliciesAndBinds.priority_analysis_results + if orig_bind_str in res and "global_type" in res[orig_bind_str]: + result = res[orig_bind_str]["global_type"] + return result + + +# store all policies and any associated binds for analysis +pols_binds = PoliciesAndBinds() diff --git a/nspepi/nspepi2/nspepi_helper b/nspepi/nspepi2/nspepi_helper new file mode 100755 index 0000000..d8be9d4 --- /dev/null +++ b/nspepi/nspepi2/nspepi_helper @@ -0,0 +1,1075 @@ +#!/usr/bin/perl -w + +# Copyright 2021 Citrix Systems, Inc. All rights reserved. +# Use of this software is governed by the license terms, if any, +# which accompany or are included with this software. + +# Warnings +# +# The tool will not optimize the configuration instead it will create equivalents only. +# +# The upgraded command might have more than 8191 chars based on the +# length of the classic command. These commands will fail while adding +# on ADC +# +# For better performance and to take full advantage of advanced syntax, +# manually we have to optimize after conversion. + +use POSIX; +use Switch; +use Getopt::Std; +use strict; +use List::Util qw[min max]; + +# CFILE - Input file +# LFILE - File containing converted commands +# WFILE - Warning file + +my ( + $classic, + $advanced, + $hash_key, + $hash_value, + $line, + $lineno, + $expression, + @expressions, + $expr_type, + $file, + %options, + $expr_params, + $q_start, + $q_end, + @nsconf, + $key, + $value, + @maxlen_warn, + $CFILE, + $LFILE, + $WFILE, + $ncfg, + $tmpcfg, + $wf, + $tmp, + $expr_converted +); + +my $ERR_BLOCKED_MSG = "Expression is in blocked list of conversion"; +my $ERR_SYNTAX_MSG = "Expression is not in proper syntax"; +my $ERR_SUPPORT_MSG = "Expression is not supported"; +my $ERR_OPERATOR_MSG = "Operator used in expression is not supported"; +my $ERR_DATE_MSG = "Date format is invaild"; + +# List of classic expressions that are convertable +my %EXPR_LIST=( + "REQ.HTTP.HEADER " , 'Type=1;expr=HTTP.REQ.HEADER("$")', + "RES.HTTP.HEADER " , 'Type=1;expr=HTTP.RES.HEADER("$")', + "HEADER " , 'Type=1;expr=HTTP.REQ.HEADER("$")', + "RES.HTTP.STATUSCODE " , 'Type=3;expr=HTTP.RES.STATUS', + "STATUSCODE " , 'Type=3;expr=HTTP.RES.STATUS', + "REQ.HTTP.URLQUERYLEN " , 'Type=5;expr=HTTP.REQ.URL.QUERY.LENGTH.GT($)', + "URLQUERYLEN " , 'Type=5;expr=HTTP.REQ.URL.QUERY.LENGTH.GT($)', + "REQ.HTTP.URLQUERY " , 'Type=2;expr=HTTP.REQ.URL.QUERY', + "URLQUERY " , 'Type=2;expr=HTTP.REQ.URL.QUERY', + "REQ.HTTP.URLLEN " , 'Type=5;expr=HTTP.REQ.URL.LENGTH.GT($)', + "URLLEN " , 'Type=5;expr=HTTP.REQ.URL.LENGTH.GT($)', + "REQ.HTTP.URL " , 'Type=7;expr=HTTP.REQ.URL', + "URL " , 'Type=7;expr=HTTP.REQ.URL', + "REQ.HTTP.METHOD " , 'Type=4;expr=HTTP.REQ.METHOD.EQ($)', + "METHOD " , 'Type=4;expr=HTTP.REQ.METHOD.EQ($)', + "LOCATION " , 'Type=4;expr=CLIENT.IP.SRC.MATCHES_LOCATION("$")', + "REQ.HTTP.VERSION " , 'Type=2;expr=HTTP.REQ.VERSION', + "VERSION " , 'Type=2;expr=HTTP.REQ.VERSION', + "RES.HTTP.VERSION " , 'Type=2;expr=HTTP.RES.VERSION', + "REQ.SSL.CLIENT.CERT.SERIALNUMBER " , 'Type=2;expr=CLIENT.SSL.CLIENT_CERT.SERIALNUMBER', + "CLIENT.CERT.SERIALNUMBER " , 'Type=2;expr=CLIENT.SSL.CLIENT_CERT.SERIALNUMBER', + "REQ.SSL.CLIENT.CERT.SUBJECT " , 'Type=2;expr=CLIENT.SSL.CLIENT_CERT.SUBJECT', + "CLIENT.CERT.SUBJECT " , 'Type=2;expr=CLIENT.SSL.CLIENT_CERT.SUBJECT', + "REQ.SSL.CLIENT.CIPHER.BITS " , 'Type=3;expr=CLIENT.SSL.CIPHER_BITS', + "CLIENT.CIPHER.BITS " , 'Type=3;expr=CLIENT.SSL.CIPHER_BITS', + "REQ.SSL.CLIENT.CERT.ISSUER " , 'Type=2;expr=CLIENT.SSL.CLIENT_CERT.ISSUER', + "CLIENT.CERT.ISSUER " , 'Type=2;expr=CLIENT.SSL.CLIENT_CERT.ISSUER', + "REQ.SSL.CLIENT.SSL.VERSION " , 'Type=3;expr=CLIENT.SSL.VERSION', + "CLIENT.SSL.VERSION " , 'Type=3;expr=CLIENT.SSL.VERSION', + "REQ.SSL.CLIENT.CERT.SIGALGO " , 'Type=2;expr=CLIENT.SSL.CLIENT_CERT.SIGNATURE_ALGORITHM', + "CLIENT.CERT.SIGALGO " , 'Type=2;expr=CLIENT.SSL.CLIENT_CERT.SIGNATURE_ALGORITHM', + "REQ.SSL.CLIENT.CERT " , 'Type=2;expr=CLIENT.SSL.CLIENT_CERT', + "CLIENT.CERT " , 'Type=2;expr=CLIENT.SSL.CLIENT_CERT', + "REQ.SSL.CLIENT.CERT.VERSION " , 'Type=3;expr=CLIENT.SSL.CLIENT_CERT.VERSION', + "CLIENT.CERT.VERSION " , 'Type=3;expr=CLIENT.SSL.CLIENT_CERT.VERSION', + "REQ.SSL.CLIENT.CIPHER.TYPE == Export" , 'Type=13;expr=CLIENT.SSL.CIPHER_EXPORTABLE', + "REQ.SSL.CLIENT.CIPHER.TYPE != Export" , 'Type=13;expr=CLIENT.SSL.CIPHER_EXPORTABLE.NOT', + "REQ.SSL.CLIENT.CERT.VALIDFROM " , 'Type=12;expr=CLIENT.SSL.CLIENT_CERT.VALID_NOT_BEFORE', + "REQ.SSL.CLIENT.CERT.VALIDTO " , 'Type=12;expr=CLIENT.SSL.CLIENT_CERT.VALID_NOT_AFTER', + "REQ.IP.DESTIP " , 'Type=6;expr=CLIENT.IP.DST', + "DESTIP " , 'Type=6;expr=CLIENT.IP.DST', + "REQ.IP.SOURCEIP " , 'Type=6;expr=CLIENT.IP.SRC', + "SOURCEIP " , 'Type=6;expr=CLIENT.IP.SRC', + "RES.IP.DESTIP " , 'Type=6;expr=SERVER.IP.DST', + "RES.IP.SOURCEIP " , 'Type=6;expr=SERVER.IP.SRC', + "MSS " , 'Type=3;expr=CLIENT.TCP.MSS', + "REQ.ETHER.DESTMAC " , 'Type=8;expr=CLIENT.ETHER.DSTMAC', + "DESTMAC " , 'Type=8;expr=CLIENT.ETHER.DSTMAC', + "REQ.ETHER.SOURCEMAC " , 'Type=8;expr=CLIENT.ETHER.SRCMAC', + "SOURCEMAC " , 'Type=8;expr=CLIENT.ETHER.SRCMAC', + "RES.ETHER.DESTMAC " , 'Type=8;expr=SERVER.ETHER.DSTMAC', + "RES.ETHER.SOURCEMAC " , 'Type=8;expr=SERVER.ETHER.SRCMAC', + "REQ.INTERFACE.DESTMAC " , 'Type=8;expr=CLIENT.ETHER.DSTMAC', + "REQ.INTERFACE.SOURCEMAC " , 'Type=8;expr=CLIENT.ETHER.SRCMAC', + "RES.INTERFACE.DESTMAC " , 'Type=8;expr=SERVER.ETHER.DSTMAC', + "RES.INTERFACE.SOURCEMAC " , 'Type=8;expr=SERVER.ETHER.SRCMAC', + "REQ.TCP.MSS " , 'Type=3;expr=CLIENT.TCP.MSS', + "REQ.TCP.DESTPORT " , 'Type=14;expr=CLIENT.TCP.DSTPORT', + "DESTPORT " , 'Type=14;expr=CLIENT.TCP.DSTPORT', + "REQ.TCP.SOURCEPORT " , 'Type=14;expr=CLIENT.TCP.SRCPORT', + "SOURCEPORT " , 'Type=14;expr=CLIENT.TCP.SRCPORT', + "RES.TCP.MSS " , 'Type=3;expr=SERVER.TCP.MSS', + "RES.TCP.DESTPORT " , 'Type=14;expr=SERVER.TCP.DSTPORT', + "RES.TCP.SOURCEPORT " , 'Type=14;expr=SERVER.TCP.SRCPORT', + "REQ.INTERFACE.ID " , 'Type=2;expr=CLIENT.INTERFACE.ID', + "ID " , 'Type=2;expr=CLIENT.INTERFACE.ID', + "REQ.INTERFACE.RXTHROUGHPUT " , 'Type=3;expr=CLIENT.INTERFACE.RXTHROUGHPUT', + "RXTHROUGHPUT " , 'Type=3;expr=CLIENT.INTERFACE.RXTHROUGHPUT', + "REQ.INTERFACE.TXTHROUGHPUT " , 'Type=3;expr=CLIENT.INTERFACE.TXTHROUGHPUT', + "TXTHROUGHPUT " , 'Type=3;expr=CLIENT.INTERFACE.TXTHROUGHPUT', + "REQ.INTERFACE.RXTXTHROUGHPUT " , 'Type=3;expr=CLIENT.INTERFACE.RXTXTHROUGHPUT', + "RXTXTHROUGHPUT " , 'Type=3;expr=CLIENT.INTERFACE.RXTXTHROUGHPUT', + "REQ.ETHER.ID " , 'Type=2;expr=CLIENT.INTERFACE.ID', + "REQ.ETHER.RXTHROUGHPUT " , 'Type=3;expr=CLIENT.INTERFACE.RXTHROUGHPUT', + "REQ.ETHER.TXTHROUGHPUT " , 'Type=3;expr=CLIENT.INTERFACE.TXTHROUGHPUT', + "REQ.ETHER.RXTXTHROUGHPUT " , 'Type=3;expr=CLIENT.INTERFACE.RXTXTHROUGHPUT', + "REQ.VLANID " , 'Type=3;expr=CLIENT.VLAN.ID', + "VLANID " , 'Type=3;expr=CLIENT.VLAN.ID', + "RES.INTERFACE.ID " , 'Type=2;expr=SERVER.INTERFACE.ID', + "RES.INTERFACE.RXTHROUGHPUT " , 'Type=3;expr=SERVER.INTERFACE.RXTHROUGHPUT', + "RES.INTERFACE.TXTHROUGHPUT " , 'Type=3;expr=SERVER.INTERFACE.TXTHROUGHPUT', + "RES.INTERFACE.RXTXTHROUGHPUT " , 'Type=3;expr=SERVER.INTERFACE.RXTXTHROUGHPUT', + "RES.ETHER.ID " , 'Type=2;expr=SERVER.INTERFACE.ID', + "RES.ETHER.RXTHROUGHPUT " , 'Type=3;expr=SERVER.INTERFACE.RXTHROUGHPUT', + "RES.ETHER.TXTHROUGHPUT " , 'Type=3;expr=SERVER.INTERFACE.TXTHROUGHPUT', + "RES.ETHER.RXTXTHROUGHPUT " , 'Type=3;expr=SERVER.INTERFACE.RXTXTHROUGHPUT', + "RES.VLANID " , 'Type=3;expr=SERVER.VLAN.ID', + "DAYOFWEEK " , 'Type=9;expr=SYS.TIME.WEEKDAY', + "DATE " , 'Type=10;expr=SYS.TIME', + "TIME " , 'Type=11;expr=SYS.TIME', +); + +# List of classic expressions that are not convertable +my @BLOCKED_LIST=( + 'CLIENT.APPLICATION', + 'CLIENT.FILE', + 'CLIENT.OS', + 'CLIENT.REG', + 'CLIENT.SVC', + 'FS.COMMAND', + 'FS.DIR', + 'FS.DOMAIN', + 'FS.FILE', + 'FS.PATH', + 'FS.SERVER', + 'FS.SERVERIP', + 'FS.SERVICE', + 'FS.USER', + 'f_5_TrendMicroOfficeScan_7_3', + 'f_5_sygate_5_6', + 'f_5_zonealarm_6_5', + 's_5_norton', + 's_farclient', + 'v_5_McAfeevirusscan_11', + 'v_5_Mcafee', + 'v_5_Symantec_10', + 'v_5_Symantec_6_0', + 'v_5_Symantec_7_5', + 'v_5_TrendMicroOfficeScan_7_3', + 'v_5_TrendMicro_11_25', + 'v_5_sophos_4', + 'v_5_sophos_5', + 'v_5_sophos_6', +); + +# List of command to do the conversion +my @CMD_LIST=( + "add policy expression (\\S+) (.+)", +); + +# List of advanced expression +my @ADVANCED_EXP_LIST=( + "AAA", + "ANALYTICS", + "CLIENT", + "CONNECTION", + "DIAMETER", + "DNS", + "FALSE", + "HTTP", + "ICA", + "MSSQL", + "MYSQL", + "ORACLE", + "RADIUS", + "SERVER", + "SIP", + "SMPP", + "SUBSCRIBER", + "SYS", + "TARGET", + "TEXT", + "TRUE", +); + +# Subroutine for printing +sub print_it(@) { + my $msg = $_[1]; + print $msg; +} + +# Main starts here +getopts("e:", \%options) or usage(); + +# If -e specified add dummy command to use the same code path to +# convert +$lineno = 0; +$expression = $options{e}; +$line = qq/add policy expression test "$expression"/; +$classic = $line; +&convertExpression; + +if ($expr_converted == 0) { + print_it($LFILE,"INFO: Expression is not converted - most likely it is a valid advanced expression\n"); +} + +if ($#maxlen_warn >= 0) { + print_it($LFILE,"WARNING: Total number of warnings due to expressions length greater than 8191 characters: ".($#maxlen_warn+1)."\n"); + print_it($LFILE,"WARNING: Line numbers which has more than 8191 characters length: ".join(', ',@maxlen_warn)."\n"); +} +if (defined $LFILE) { + print_it($LFILE,"OUTPUT: New configuration file created: $ncfg\n"); + print_it($LFILE,"OUTPUT: New warning file created: $wf\n"); +} +if (defined $LFILE) { + close($LFILE); +} +if (defined $WFILE) { + close($WFILE); +} + +exit(0); + +# Error handling +sub err_handle +{ + print STDERR "ERROR: @_\n"; +} + +# Subroutine to replace the entity names like expressions +sub intelligentReplacer +{ + my $input = $_[0]; + my $find = "\Q$_[1]\E"; + my $replace = $_[2]; + + # Check for the complex patterns + if ($input =~/(&&|\|\|)/) { + # Fix begining + $input =~ s/(test \"\s*[\(|!]*\s*)$find(\s*\)*\s*[&&|\|\|])/$1$replace$2/; + # Fix ending + $input =~ s/([&&|\|\|]\s*[\(|!]*\s*)$find(\s*\)*\s*\")$/$1$replace$2/; + # Fix middle + $input =~ s/([&&|\|\|]\s*[\(|!]*\s*)$find(\s*\)*\s*[&&|\|\|])/$1$replace$2/g; + } + else { + $input =~ s/$find/$replace/; + } + return $input; +} +# Main function to convert a single command +sub convertExpression +{ + + $expression= ""; + $expr_type= ""; + my $exp = ""; + # Set it 1 here so that in case + # of error we don't log + # information message. + $expr_converted = 1; + + # Extract the expression from command line. Print others as it is + if (!extractExpressions()) { + print_it($LFILE,"$line\n"); + err_handle $ERR_SUPPORT_MSG; + return; + } + # Check for blocked expressions + foreach (@BLOCKED_LIST) { + if ($expression =~ /$_/i) { + print_it($LFILE,"$line\n"); + err_handle $ERR_BLOCKED_MSG; + return; + } + } + # Set it 0 after handling + # error cases, so that we + # log the message only if + # expression is not converted. + $expr_converted = 0; + #skip conversion advanced policies + foreach my $adv_cmd (@ADVANCED_EXP_LIST) + { + if ($expression =~ m/^["!\d(\s]*$adv_cmd/i) { + goto SKIP; + } + } + # Convert classic into advanced + foreach $exp (@expressions) { + + foreach $hash_key (keys(%EXPR_LIST)) { + if ($exp =~ /^\((.*)\)$/s) { + $exp = $1; + } + if ($exp =~ /^$hash_key(.*)$/is) { + $expr_params = $1; + $hash_value = $EXPR_LIST{$hash_key}; + $hash_value =~ /Type=(\d+);expr=(.+)/s; + $advanced = $2; + $expr_type = $1; + identifyExpressions(); + $classic = intelligentReplacer($classic, $exp, $advanced); + $expr_converted = 1; + + #Fix to Handle cases where arguments are not specified like REQ.IP.SOURCEIP + } elsif (uc($hash_key) eq (uc($exp) . " ")) { + $hash_value = $EXPR_LIST{$hash_key}; + $hash_value =~ /Type=(\d+);expr=(.+)/s; + $advanced = $2; + $classic = intelligentReplacer($classic, $exp, $advanced); + $expr_converted = 1; + } + } + } +SKIP: + if (length($classic) > 1100) { + my $dl = length($classic)-length($line); + if ($dl > 300) { + push @maxlen_warn, $lineno; + } + } + #Removing the dummy expression created for -e + $classic =~ s/^\badd\b\s*\bpolicy\b\s*\bexpression\b\s*\btest\b\s*//; + print_it($LFILE,"$classic\n"); +} + +sub trim($) +{ + my $text = shift; + $text =~ s/^\s+//; + $text =~ s/\s+$//; + return $text; +} + +# Subroutine to remove the quotes +sub removeQuote +{ + my ($qs, $qe); + my $text = $_[0]; + $text = trim($text); + + $qs = substr $text, 0, 1; + if ($qs =~ /q/) { + $qs = substr $text, 0, 2; + $qe = substr $text, 1,1; + } + elsif ($qs =~ /\\/) { + $qs = substr $text, 0, 2; + $qe = $qs; + } + elsif ($qs =~ /\(/) { + $qs=""; + $qe=""; + } + elsif ($qs =~ /\"/) { + $qe = $qs; + } + else { + $qs = ""; + $qe = ""; + } + $text = substr($text, length($qs)); + $text = substr($text, 0,length($text)-length($qe)); + $q_start = $qs; + return $text; +} +# Extract the individual classic expressions from the given command +sub extractExpressions +{ + my $cmd; + my $exp; + my $tmp; + my $f = 0; + my $and = "&&"; + my $or = "||"; + @expressions = (); + + # Process the command list + foreach $cmd (@CMD_LIST) { + + if ($classic =~ /^$cmd$/is) { + $expression = $2; + + $expression = removeQuote($expression); + $exp = $expression; + if ($exp =~ /($and|$or)/) { + if ($exp =~ /\(|!/) { + $exp =~ s/^(\(|!)+//; + $exp =~ s/\)+$//; + $exp =~ s/\)+\s*$and/ $and/g; + $exp =~ s/$and\s*(\(|!)+/$and /g; + $exp =~ s/\)+\s*\Q$or\E/ $or/g; + $exp =~ s/\Q$or\E\s*(\(|!)+/$or /g; + } + } + @expressions = split(/(\s*$and\s*|\s*\Q$or\E\s*)/,$exp); + @expressions = grep(!/(\s*$and\s*|\s*\Q$or\E\s*)/,@expressions); + + foreach $exp (@expressions) { + $exp = trim($exp); + } + + return 1; + } + } +} + +# Subroutine to identify the type of the expressions +sub identifyExpressions +{ + my $header_name=""; + my $OPERATOR=""; + my $ARG2=""; + my $qu='"'; + + if ($q_start eq "\"") { + $qu = "\\\""; + } + + switch($expr_type) { + #1) HTTP Header Based Expressions. Example: "REQ.HTTP.HEADER Accept-Language == en-us" + case 1 { + if ($expr_params =~ /^(\S+)(\s+.*)$/s) { + $header_name=$1; + $expr_params=$2; + $advanced =~ s/"\$"/$qu$header_name$qu/; + if ($expr_params =~ /^\s+(\S+)(\s*.*)$/s) { + $OPERATOR=$1; + $expr_params=$2; + parseTextOperator($OPERATOR); + return; + } + } + else { + err_handle $ERR_SYNTAX_MSG; + return; + } + } + #2) TEXT Based Expressions. + case 2 { + if ($expr_params =~ /^(\S+)(\s*.*)$/s) { + my $OPERATOR=$1; + $expr_params=$2; + parseTextOperator($OPERATOR); + return; + } + else { + err_handle $ERR_SYNTAX_MSG; + return; + } + } + #3) Number Based Expressions. + case 3 { + if ($expr_params =~ /^(\S+)\s+(\d+)(.*)$/s) { + my $operator=$1; + my $argument=$2; + $expr_params=$3; + parseNumericOperator($operator,$argument); + return; + } + else { + err_handle $ERR_SYNTAX_MSG; + return; + } + } + #4) TEXT based substitution. Replace $. + case 4 { + if ($expr_params =~ /^(\S+)(\s*.*)$/s) { + my $OPERATOR=$1; + $expr_params=$2; + my $ARG = extractArgument(); + if ($OPERATOR eq '==') { + if ($advanced =~ /"\$"/) { + $advanced =~ s/"\$"/$qu$ARG$qu/; + } + else { + $advanced =~ s/\$/$ARG/; + } + } + elsif ($OPERATOR eq '!=') { + if ($advanced =~ /"\$"/) { + $advanced =~ s/"\$"/$qu$ARG$qu/; + } + else { + $advanced =~ s/\$/$ARG/; + } + $advanced = $advanced .'.NOT'; + } + else { + err_handle $ERR_OPERATOR_MSG; + return; + } + return; + } + else { + err_handle $ERR_SYNTAX_MSG; + return; + } + } + #5) Number based substitution. Replace $. + case 5 { + if ($expr_params =~ /^>\s+(\d+)(.*)$/s) { + my $argument=$1; + $expr_params=$2; + $advanced =~ s/\$/$argument/; + return; + } + else { + err_handle $ERR_SYNTAX_MSG; + return; + } + } + #6) IP Based Expressions. + case 6 { + if ($expr_params =~ /^(\S\S)\s+(\d+\.\d+\.\d+.\d+)(.*)$/s) { + my $operator = $1; + my $ip_addr = $2; + $expr_params = $3; + + #Check for netmask# + if ($expr_params =~ /^\s+-netmask\s+(\d+\.\d+\.\d+.\d+)(.*)$/is) { + my $netmask = $1; + $expr_params=$2; + #Convert netmask for PI syntax. + $netmask = parseNetmask($netmask); + $advanced = $advanced.'.IN_SUBNET('.$ip_addr.'/'.$netmask.')'; + } + else { + $advanced = $advanced.'.EQ('.$ip_addr.')'; + } + if ($operator eq '!=') { + $advanced =$advanced.'.NOT'; + } + elsif ($operator ne '==') { + err_handle $ERR_OPERATOR_MSG; + return; + } + return; + } + else { + err_handle $ERR_SUPPORT_MSG; + return; + } + } + #7) URL based Expression.This case is diferent because of * char. in argument. + case 7 { + if ($expr_params =~ /^(\S+)(\s*.*)$/s) { + my $OPERATOR=$1; + my $expression_temp=$expr_params=$2; + + if (($OPERATOR ne '==') && ($OPERATOR ne '!=')) { + parseTextOperator($OPERATOR); + return; + } + else { + my $ARG = extractArgument(); + if ($ARG !~ /\*/) { + $expr_params=$expression_temp; + parseTextOperator($OPERATOR); + return; + } + $ARG =~ s/\./\\./; + $ARG =~ s/\*/(.*)/g; + $advanced = $advanced.'.REGEX_MATCH(re#'.$ARG.'#)'; + if ($OPERATOR eq '!=') { + $advanced =$advanced.'.NOT'; + } + return; + } + } + else { + err_handle $ERR_SUPPORT_MSG; + return; + } + } + #8) MAC Based Expressions. + case 8 { + my $mac_addr = q/[0-9a-fA-f]{2}:[0-9a-fA-f]{2}:[0-9a-fA-f]{2}:[0-9a-fA-f]{2}:[0-9a-fA-f]{2}:[0-9a-fA-f]{2}/; + if ($expr_params =~ /^(\S\S)\s+($mac_addr)(.*)$/s) { + my $operator = $1; + $mac_addr= $2; + $expr_params = $3; + $advanced = $advanced.'.EQ('.$mac_addr.')'; + + if ($operator eq '!=') { + $advanced =$advanced.'.NOT'; + } + elsif ($operator ne '==') { + err_handle $ERR_OPERATOR_MSG; + return; + } + return; + } + else { + err_handle $ERR_SUPPORT_MSG; + return; + } + } + #9) "DAYOFWEEK" Expression. Example:- "DAYOFWEEK == SUNDAYGMT" + case 9 { + if ($expr_params =~ /^(\S{1,2})\s+(.+)GMT(.*)$/is) { + my $operator = $1; + my $day= $2; + $expr_params = $3; + my %weekday = (Sunday => 0, Monday => 1, Tuesday => 2, Wednesday => 3, Thursday => 4, Friday => 5, Saturday => 6); + $day = ucfirst (lc $day); + $day = $weekday{$day}; + if (!defined $day) { + err_handle $ERR_DATE_MSG; + return; + } + parseNumericOperator($operator,$day); + } + else { + err_handle $ERR_DATE_MSG; + return; + } + } + #10) "DATE" Expression. Example:- "DATE == '2010-05-24GMT'" + case 10 { + if ($expr_params =~ /^(\S+)(\s*.*)$/s) { + my $OPERATOR=$1; + $expr_params=$2; + my %month = ( + '01' => "Jan", + '02' => "Feb", + '03' => "Mar", + '04' => "Apr", + '05' => "May", + '06' => "Jun", + '07' => "Jul", + '08' => "Aug", + '09' => "Sep", + '10' => "Oct", + '11' => "Nov", + '12' => "Dec" + ); + my $ARG = extractArgument(); + + if (uc($OPERATOR) eq "BETWEEN") { + if ($ARG =~ /^(\d{4})-(\d{2})-(\d{2})(\S{3})-(\d{4})-(\d{2})-(\d{2})(\S{3})$/) { + $ARG = "$4 $1 $month{$2} $3"; + $ARG2 = "$8 $5 $month{$6} $7"; + $advanced = $advanced . ".BETWEEN($ARG,$ARG2)"; + return; + } + else { + err_handle $ERR_DATE_MSG; + return; + } + } + else { + if ($ARG =~ /^(\d{4})-(\d{2})-(\d{2})(\S{3})$/) { + $ARG = "$4 $1 $month{$2} $3"; + parseNumericOperator($OPERATOR,$ARG); + return; + } + else { + err_handle $ERR_OPERATOR_MSG; + return; + } + } + } + else { + err_handle $ERR_SYNTAX_MSG; + return; + } + } + + #11) "TIME" Expression. Example:- "TIME != \'2010-05-26-13:31:37GMT\'" + case 11 { + if ($expr_params =~ /^(\S+)(\s*.*)$/s) { + my $OPERATOR=$1; + $expr_params=$2; + my %month = ( + '01' => "Jan", + '02' => "Feb", + '03' => "Mar", + '04' => "Apr", + '05' => "May", + '06' => "Jun", + '07' => "Jul", + '08' => "Aug", + '09' => "Sep", + '10' => "Oct", + '11' => "Nov", + '12' => "Dec" + ); + my $ARG = extractArgument(); + if (uc($OPERATOR) eq "BETWEEN") { + if ($ARG =~ /^(\d{4})-(\d{2})-(\d{2})-(\d{2}):(\d{2}):(\d{2})(\S{3})-(\d{4})-(\d{2})-(\d{2})-(\d{2}):(\d{2}):(\d{2})(\S{3})$/) { + $ARG = "$7 $1 $month{$2} $3 $4h $5m $6s"; + $ARG2 = "$14 $8 $month{$9} $10 $11h $12m $13s"; + $advanced = $advanced . ".BETWEEN($ARG,$ARG2)"; + return; + } + else { + err_handle $ERR_DATE_MSG; + return; + } + } + else { + if ($ARG =~ /^(\d{4})-(\d{2})-(\d{2})-(\d{2}):(\d{2}):(\d{2})(\S{3})$/) { + $ARG = "$7 $1 $month{$2} $3 $4h $5m $6s"; + parseNumericOperator($OPERATOR,$ARG); + return; + } + else { + err_handle $ERR_DATE_MSG; + return; + } + } + } + else { + err_handle $ERR_SYNTAX_MSG; + return; + } + } + #12) Diffrent "Time" format used in Client-Cert Expressions. Example: 'Thu, 27 May 2010 16:10:15 GMT' + case 12 { + if ($expr_params =~ /^(\S+)(\s*.*)$/s) { + my $OPERATOR=$1; + $expr_params=$2; + my $ARG = extractArgument(); + if ($ARG =~ /^\S{3}\, (\d{2}) (\S{3}) (\d{4}) (\d{2}):(\d{2}):(\d{2}) (\S{3})$/) { + $ARG = "$7 $3 $2 $1 $4h $5m $6s"; + parseNumericOperator($OPERATOR,$ARG); + return; + } + else { + err_handle $ERR_OPERATOR_MSG; + return; + } + } + else { + err_handle $ERR_SYNTAX_MSG; + return; + } + } + #13) Simple replacement with Hash value. + case 13 { + if ($qu ne "\"") { + $advanced =~ s/\(\"/\($qu/g; + $advanced =~ s/\"\)/$qu\)/g; + } + return; + } + #14) Port Based Expressions. + case 14 { + if ($expr_params =~ /^(\S+)\s+(\d+)(.*)$/s) { + my $operator=$1; + my $argument=$2; + $expr_params=$3; + if ($expr_params =~ /^-/) { + parsePortRangeExpr($operator,$argument); + } else { + parseNumericOperator($operator,$argument); + } + return; + } + else { + err_handle $ERR_SYNTAX_MSG; + return; + } + } + } +} + +# Subroutine to extract the argument value from expression +sub extractArgument +{ + if ($expr_params =~ m/^\s+(\S+)(\s*.*)$/s) { + my $temp1= $1; + $expr_params=$2; + my $arg_value = $temp1; + if ($arg_value =~ /^(\\\'|\\\"|\"|\')(.*)/s) { + my $quote_char = $1; + $arg_value = $2; + my $flag = 1; + $quote_char = '\\'.$quote_char; + + if ($arg_value =~ /(.*?)$quote_char(.*)/s) { + my $temp = $arg_value = $1;$flag = 0; + $expr_params=$2.$expr_params; + if (chop($temp) eq "\\") { + $arg_value = $arg_value.'\\'.$quote_char; + $flag = 1; + } + } +LEV_1: + if (($expr_params =~ /(.*?)$quote_char(.*)/s) && $flag) { + my $temp = $arg_value = $arg_value.$1; + $expr_params=$2; + if (chop($temp) eq "\\") { + $arg_value = $arg_value.'\\'.$quote_char; + goto LEV_1; + } + } + } + else { + if ($arg_value =~ /^(.+?)(\).*)/s) { + $arg_value = $1; + $expr_params = $2.$expr_params; + } + } + return $arg_value; + } +} + +# Subroutine to find the text operation's equivlent +sub parseTextOperator +{ + my $operator = shift; + my $qu='"'; + + if ($q_start eq "\"") { + $qu = "\\\""; + } + switch($operator) { + case '==' { + my $ARG = extractArgument(); + $advanced = $advanced . '.EQ('.$qu.$ARG.$qu.')'; + return; + } + case '!=' { + my $ARG = extractArgument(); + $advanced = $advanced . '.EQ('.$qu.$ARG.$qu.').NOT'; + } + case (/^EXISTS.*/is) { + if ($operator=~ /EXISTS(.*)/is) { + $expr_params = $1.$expr_params; + } + $advanced = $advanced . '.EXISTS'; + return; + } + case (/^NOTEXISTS.*/is) { + if ($operator=~ /NOTEXISTS(.*)/is) { + $expr_params = $1.$expr_params; + } + $advanced = $advanced . '.EXISTS.NOT'; + return; + } + case /CONTAINS$/i { + my $ARG = extractArgument(); + my $advanced2 ='.CONTAINS('.$qu.$ARG.$qu.')'; + my $flag = 0; + # To handle the LENGTH and OFFSET params. +LEN_OR_OFF: + if ($expr_params =~ /^\s+(-l\S*|-o\S*)\s+(\d+)(.*)$/is) { + my $temp1=$1; + my $temp2=$2; + my $temp3=$3; + if (length($temp1) > 7) { + err_handle $ERR_SYNTAX_MSG; + return; + } + if ((lc($temp1) eq substr("-length",0,length($temp1))) && (($flag & 1) == 0)) { + $advanced2 = '.SUBSTR(0,'.$temp2.')'.$advanced2; + $expr_params = $temp3; + $flag = $flag | 1; + goto LEN_OR_OFF; + } + if ((lc($temp1) eq substr("-offset",0,length($temp1))) && (($flag & 2) == 0)) { + $advanced2 = '.SKIP('.$temp2.')'.$advanced2; + $expr_params = $temp3; + $flag = $flag | 2; + goto LEN_OR_OFF; + } + } + $advanced = $advanced . $advanced2; + if ($operator =~ /^NOTCONTAINS$/i) { + $advanced = $advanced .'.NOT'; + } + return; + } + case /^CONTENTS$/i { + my $advanced2 =""; + my $flag = 0; + # To handle the LENGTH and OFFSET params. +LEN_OR_OFF2: + if ($expr_params =~ /^\s+(-l\S*|-o\S*)\s+(\d+)(.*)$/is) { + my $temp1=$1; + my $temp2=$2; + my $temp3=$3; + if (length($temp1) > 7) { + err_handle $ERR_SYNTAX_MSG; + return; + } + if ((lc($temp1) eq substr("-length",0,length($temp1))) && (($flag & 1) == 0)) { + $advanced2 = '.SUBSTR(0,'.$temp2.')'; + $expr_params = $temp3; + $flag = $flag | 1; + goto LEN_OR_OFF2; + } + if ((lc($temp1) eq substr("-offset",0,length($temp1))) && (($flag & 2) == 0)) { + $advanced = $advanced.'.SKIP('.$temp2.')'; + $expr_params = $temp3; + $flag = $flag | 2; + goto LEN_OR_OFF2; + } + } + $advanced = $advanced . $advanced2.".LENGTH.GT(0)"; + return; + } + else { + err_handle $ERR_OPERATOR_MSG; + return; + } + } +} + +# Subroutine to parse port range expression +sub parsePortRangeExpr +{ + my $operator = shift; + my $ARG = shift; + my $ARG2; + + if ($expr_params =~ m/^-(\d+)(.*)$/s) { + $ARG2 = $1; + $expr_params = $2; + } + else { + err_handle $ERR_SYNTAX_MSG; + return; + } + switch($operator) { + case '==' { + $advanced = $advanced . ".BETWEEN(". min($ARG, $ARG2) . ", " . max($ARG, $ARG2) .")"; + return; + } + case '!=' { + $advanced = $advanced . ".BETWEEN(". min($ARG, $ARG2) . ", " . max($ARG, $ARG2). ").NOT"; + return; + } + case '>=' { + $advanced = $advanced . '.GE('.$ARG.')' . ' || ' . $advanced . '.GE('.$ARG2.')'; + return; + } + case '>' { + $advanced = $advanced . '.GT('.$ARG.')' . ' || ' . $advanced . '.GT('.$ARG2.')'; + return; + } + case '<=' { + $advanced = $advanced . '.LE('.$ARG.')' . ' && ' . $advanced . '.LE('.$ARG2.')'; + return; + } + case '<' { + $advanced = $advanced . '.LT('.$ARG.')' . ' && ' . $advanced . '.LT('.$ARG2.')'; + return; + } + case /^BETWEEN$/i { + $advanced = $advanced . ".BETWEEN(" . min($ARG, $ARG2). ", " . max($ARG, $ARG2) . ")"; + return; + } + else { + err_handle $ERR_OPERATOR_MSG; + return; + } + } +} + +# Subroutine to parse numeric operators +sub parseNumericOperator +{ + my $operator = shift; + my $ARG = shift; + + switch($operator) { + case '==' { + $advanced = $advanced . '.EQ('.$ARG.')'; + return; + } + case '!=' { + $advanced = $advanced . '.EQ('.$ARG.').NOT'; + return; + } + case '>=' { + $advanced = $advanced . '.GE('.$ARG.')'; + return; + } + case '>' { + $advanced = $advanced . '.GT('.$ARG.')'; + return; + } + case '<=' { + $advanced = $advanced . '.LE('.$ARG.')'; + return; + } + case '<' { + $advanced = $advanced . '.LT('.$ARG.')'; + return; + } + case /^BETWEEN$/i { + if ($expr_params =~ m/^-(\d+)(.*)$/s) { + my $ARG2 = $1; + $expr_params = $2; + $advanced = $advanced . ".BETWEEN($ARG,$ARG2)"; + return; + } + else { + err_handle $ERR_SYNTAX_MSG; + return; + } + } + else { + err_handle $ERR_OPERATOR_MSG; + return; + } + } +} + +# Subroutine to parse the sub net mask +sub parseNetmask { + my $netmask = shift; + my $bit_count=0; + my @GET; + $netmask =~ /(\d+).(\d+).(\d+).(\d+)/; + $GET[1]= $1;$GET[2]= $2;$GET[3]= $3;$GET[4]= $4; + my $i=1; + my $temp = 1<<7; + + while((($GET[$i] & $temp) != 0) && ($i <= 4)) { + $temp=$temp>>1; + if ($temp == 0) { + $temp = 1<<7;$i++; + } + $bit_count++; + } + return $bit_count; +} + +# Subroutine for usage or help +sub usage { + print "Usage: nspepi -e \"classic expression\"\n"; + print "Max expression length = 8191\n\n"; + print "Output format:\n\n"; + print "\t\n"; + exit; +} diff --git a/nspepi/nspepi2/nspepi_main.py b/nspepi/nspepi2/nspepi_main.py new file mode 100755 index 0000000..6734219 --- /dev/null +++ b/nspepi/nspepi2/nspepi_main.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python2 + +# Copyright 2021 Citrix Systems, Inc. All rights reserved. +# Use of this software is governed by the license terms, if any, +# which accompany or are included with this software. + +""" +Convert classic expressions to advanced expressions and deprecated +commands to non-deprecated ones. + +Dependency packages: PLY, pytest +""" + +# Ensure that the version string conforms to PEP 440: +# https://www.python.org/dev/peps/pep-0440/ +__version__ = "1.0" + +import argparse +import glob +import importlib +import logging +import logging.handlers +import os +import os.path +import sys +from inspect import cleandoc +import inspect +import re + +import cli_yacc +from convert_classic_expr import convert_classic_expr, \ + convert_adv_expr +import nspepi_common as common + +import convert_cli_commands + +# Log handlers that need to be saved from call to call +file_log_handler = None +console_log_handler = None +debug_log_handler = None + +def create_file_log_handler(file_name, log_level): + """ + Creates file logging handler. + + Args: + file_name - log file name + log_level - The level of logs to put in the file + """ + # create file handler and roll logs if needed + exists = os.path.isfile(file_name) + file_handler = logging.handlers.RotatingFileHandler(file_name, + mode='a', + backupCount=9) + if exists: + file_handler.doRollover() + # set the file log handler level + file_handler.setLevel(log_level) + # create formatters and add them to the handlers + fh_format = logging.Formatter('%(asctime)s: %(levelname)s - %(message)s') + file_handler.setFormatter(fh_format) + return file_handler + +def setup_logging(log_file_name, file_log_level, debug_file_name, console_output_needed): + """ + Sets up logging for the program. + + Args: + log_file_name: The name of the log file + file_log_level: The level of logs to put in log_file_name file + debug_file_name: The name of the debug log file + console_output_needed: True if logs need to be seen on console + """ + global file_log_handler + global console_log_handler + global debug_log_handler + # create logger + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + # if called multiple times, remove existing handlers + logger.removeHandler(file_log_handler) + logger.removeHandler(console_log_handler) + logger.removeHandler(debug_log_handler) + # create file handler + file_log_handler = create_file_log_handler(log_file_name, file_log_level) + # add the handlers to the logger + logger.addHandler(file_log_handler) + if debug_file_name: + debug_log_handler = create_file_log_handler(debug_file_name, logging.DEBUG) + logger.addHandler(debug_log_handler) + if console_output_needed: + # create console handler that sees even info messages + console_log_handler = logging.StreamHandler() + console_log_handler.setLevel(logging.INFO) + ch_format = logging.Formatter('%(levelname)s - %(message)s') + console_log_handler.setFormatter(ch_format) + logger.addHandler(console_log_handler) + + +def classic_policy_expr(expr): + """ + Validates that the length of expression given does not exceed 8191 chars. + + Args: + expr: Classic policy expression whose length is to be validated + + Returns: + expr: Classic policy expression passed-in as argument + + Raises: + argparse.ArgumentTypeError: If length of expr exceeds 8191 chars + """ + if (len(expr) > 8191): + raise argparse.ArgumentTypeError("expression length exceeds 8191" + " characters") + return expr + + +def output_line(line, outfile, verbose): + """ + Output a (potentially) converted line. + + Args: + line: the line to output + outfile: Output file to write converted commands + verbose: True iff converted commands should also be output to console + """ + outfile.write(line) + if verbose: + logging.info(line.rstrip()) + + +def convert_config_file(infile, outfile, verbose): + """ + Process ns config file passed in argument and convert classic policy + expressions to advanced expressions and deprecated commands to + non-deprecated commands. + + Args: + infile: NS config file to be converted + outfile: Output file to write converted commands + verbose: True iff converted commands should also be output to console + """ + cli_yacc.cli_yacc_init() + # import all modules that start with convert_* so that the handler methods + # for various commands are registered + currentfile = os.path.abspath(inspect.getfile(inspect.currentframe())) + currentdir = os.path.dirname(currentfile) + for module in glob.glob(os.path.join(currentdir, 'convert_*.py')): + importlib.import_module(os.path.splitext(os.path.basename(module))[0]) + # call methods registered to be called before the start of processing + # config file. + for m in common.init_methods: + m.method(m.obj) + lineno = 0 + for cmd in infile: + lineno += 1 + parsed_tree = cli_yacc.cli_yacc_parse(cmd, lineno) + if parsed_tree is not None: + # construct dictionary key to look up registered method to call to + # parse and transform the command to be emitted + # Registered method can return either string or tree. + key = " ".join(parsed_tree.get_command_type()).lower() + if key in common.dispatchtable: + for m in common.dispatchtable[key]: + for output in m.method(m.obj, parsed_tree): + output_line(str(output), outfile, verbose) + else: + output_line(str(parsed_tree), outfile, verbose) + else: + output_line(cmd, outfile, verbose) + # call methods registered to be called at end of processing + for m in common.final_methods: + for output in m.method(m.obj): + output_line(str(output), outfile, verbose) + # analyze policy bindings for any unsupported bindings + common.pols_binds.analyze() + # Get all bind commands after reprioritizing. + config_obj = convert_cli_commands.ConvertConfig() + for output in config_obj.reprioritize_and_emit_binds(): + output_line(str(output), outfile, verbose) + + +def main(): + desc = cleandoc( + """ + Convert classic policy expressions to advanced policy + expressions and deprecated commands to non-deprecated + commands. + """) + usage_example = cleandoc( + """ + Usage Examples: + i) nspepi -e "req.tcp.destport == 80" + ii) nspepi -f ns.conf + """) + arg_parser = argparse.ArgumentParser( + prog="nspepi", + description=desc, + epilog=usage_example, + formatter_class=argparse.RawDescriptionHelpFormatter) + # create a mutually exclusive group for arguments -e and -f to specify that + # only one of them can be accepted and not both at the same time. Also set + # required=True to specify that at least one of them must be given as an + # argument. + group = arg_parser.add_mutually_exclusive_group(required=True) + group.add_argument( + "-e", "--expression", action="store", type=classic_policy_expr, + metavar="", + help="convert classic policy expression to advanced policy" + " expression (maximum length of 8191 allowed)") + group.add_argument( + "-f", "--infile", metavar="", + help="convert Citrix ADC configuration file") + arg_parser.add_argument( + "-d", "--debug", action="store_true", help="log debug output") + arg_parser.add_argument( + "-v", "--verbose", action="store_true", help="show verbose output") + arg_parser.add_argument( + '-V', '--version', action='version', + version='%(prog)s {}'.format(__version__)) + try: + args = arg_parser.parse_args() + except IOError as e: + exit(str(e)) + # obtain logging parameters and setup logging + conf_file_path = '' + conf_file_name = 'expr' + if args.infile is not None: + conf_file_path = os.path.dirname(args.infile) + conf_file_name = os.path.basename(args.infile) + log_file_name = os.path.join(conf_file_path, 'warn_' + conf_file_name) + debug_file_name = os.path.join(conf_file_path, 'debug_' + conf_file_name) if args.debug else None + # For -v and -e options, logs will be seen on console and warn file. + # For other options, logs will only be in warn file and not on console. + setup_logging(log_file_name, logging.WARNING, debug_file_name, args.verbose or args.expression is not None) + convert_cli_commands.convert_cli_init() + # convert classic policy expression if given as an argument + if args.expression is not None: + # Check that given argument value is not a command + if re.search(r'^\s*((add)|(set)|(bind))\s+[a-zA-Z]', args.expression, re.IGNORECASE): + print("Error: argument e: Make sure argument value " + "provided is an expression and not a command") + return + output = convert_classic_expr(args.expression) + # return value of convert_classic_expr will be enclosed with quotes. + if output is not None and convert_cli_commands. \ + remove_quotes(output) == args.expression: + # If expression is not converted, then it can be advanced + # expression. Advanced expressions can have Q and S prefixes and + # SYS.EVAL_CLASSIC_EXPR expression which needs to be converted. + output = convert_adv_expr(args.expression) + if output is not None: + print(output) + # convert ns config file + elif args.infile is not None: + new_path = os.path.join(conf_file_path, "new_" + conf_file_name) + with open(args.infile, 'r') as infile: + with open(new_path, 'w') as outfile: + convert_config_file(infile, outfile, args.verbose) + print("\nConverted config will be available in a new file new_" + + conf_file_name + ".\nCheck warn_" + conf_file_name + + " file for any warnings or errors that might have been generated.") + if args.debug: + print("Check debug_" + conf_file_name + " file for debug logs.") + + +if __name__ == '__main__': + main() diff --git a/nspepi/nspepi2/nspepi_parse_tree.py b/nspepi/nspepi2/nspepi_parse_tree.py new file mode 100644 index 0000000..bdda52f --- /dev/null +++ b/nspepi/nspepi2/nspepi_parse_tree.py @@ -0,0 +1,539 @@ +#!/usr/bin/env python2 + +# Copyright 2021 Citrix Systems, Inc. All rights reserved. +# Use of this software is governed by the license terms, if any, +# which accompany or are included with this software. + +import re +import logging +from collections import OrderedDict + +import nspepi_common as common + + +""" +Parse tree implementation + +Example: +Assume you had the command at line 123: + add responder action foo respondwith "HTTP/1.1 403 Forbidden\r\n\r\n" -comment "My comment" +then you would get the following parse tree (note: when generating a new parse +tree that is a replacement for something in ns.conf you can just pass '' for +the original text): + +CLICommand ('add', 'responder', 'action') + | + CLIPositionalParameter ('foo') + | + CLIPositionalParameter ('respondwith') + | + CLIPositionalParameter ('"HTTP/1.1 403 Forbidden\r\n\r\n"') + | + CLIKeywordParameter + | + CLIKeywordName ('comment') [note no '-'] + | + CLIKeywordValue ('My comment') + +This tree can be created by the following code: + cmd = CLICommand('add', 'responder', 'action') + # Note the 'quoted' parameter is defaulted to False below + pp = CLIPositionalParameter('foo') + cmd.add_positional(pp) + pp = CLIPositionalParameter('respondwith') + cmd.add_positional(pp) + pp = CLIPositionalParameter(r'"HTTP/1.1 403 Forbidden\r\n\r\n"') + cmd.add_positional(pp) + kwn = CLIKeywordName('comment') + kwp = CLIKeywordParameter(kwn) + # Note the 'quoted' parameter to CLIKeywordValue is defaulted to False + kwp.add_value('My comment') + cmd.add_keyword(kwp) +""" + + +class CLIParseTreeNode(object): + """ A CLI parse tree node """ + must_quote_chars = re.compile('[ \t\r\n"\'\\\\()]') + must_escape_chars = "\t\r\n\"\\" + qquote_delims = "/{<|~$^+=&%@`?" + + def __init__(self): + """ Create a CLI parse tree node object """ + pass + + def normalize(self, val, make_str=False): + """ Normalizes the string representation for an item in a CLI command + so that it will correctly be understood by the CLI. + val - the string value to normalize. + make_str - to normalize when special characters are needed in string. + Returns the normalized string. + """ + # Only uses double quotes if any quoting is needed. + # This may put in quotes in some cases where they are not actually + # needed. + result = val + str_len = len(val) + if str_len == 0: + result = '""' + else: + if (make_str or val[0] == '-' or val[0] == '#' + or (val[0] == 'q' and str_len > 1 and + val[1] in self.qquote_delims) + or self.must_quote_chars.search(val)): + result = '"' + for ch in val: + if ch in self.must_escape_chars: + if ch == '\t': + result += '\\t' + elif ch == '\r': + result += '\\r' + elif ch == '\n': + result += '\\n' + else: + result += '\\' + ch + else: + result += ch + result += '"' + return result + + +class CLICommand(CLIParseTreeNode): + """ A CLI configuration command """ + + def __init__(self, op, group, ot): + """ Create a CLI command object + upgraded - Indicates whether the command is upgraded or not + adv_upgraded - In many places of the code, upgraded flag is used to determine if original policy is classic + or advanced. If upgraded is true, then original policy is considered to be classic. But advanced + policies can have SYS.EVAL_CLASSIC_EXPR, which needs to be converted as well. Here original + policy is advanced but need to set upgrade flag after conversion. adv_upgraded should be used + in such cases instead of upgraded. + invalid - Indicates whether the command is invalid in 13.1 release. + original_line - the text of the line that was parsed + lineno - the line number (starting with 1) that the command occurs on + op - the op-code for the command + group - the group for the command + ot - the object type + """ + self._upgraded = True + self._adv_upgraded = True + self._invalid = False + self._original_line = "" + self._lineno = 0 + self._op = op + self._group = group + self._ot = ot + self._positionals = [] + self._keywords = OrderedDict() + super(CLICommand, self).__init__() + logging.debug('CLICommand created: op=' + op + + ', group=' + group + + ', ot=' + ot) + + def get_command_type(self): + return [self._op, self._group, self._ot] + + @property + def lineno(self): + return self._lineno + + @lineno.setter + def lineno(self, lineno): + self._lineno = lineno + logging.debug('CLICommand lineno set: ' + str(lineno)) + + @property + def original_line(self): + return self._original_line + + @original_line.setter + def original_line(self, original_line): + self._original_line = original_line + self._upgraded = False + self._adv_upgraded = False + logging.debug('CLICommand original_line set: ' + original_line + + ', upgraded set to False') + + @property + def op(self): + return self._op + + @op.setter + def op(self, op): + self._op = op + logging.debug('CLICommand ot set: ' + op) + + @property + def group(self): + return self._group + + @group.setter + def group(self, group): + self._group = group + logging.debug('CLICommand ot set: ' + group) + + @property + def ot(self): + return self._ot + + @ot.setter + def ot(self, ot): + self._ot = ot + logging.debug('CLICommand ot set: ' + ot) + + def set_upgraded(self): + """ Flags that this command was upgraded. """ + self._upgraded = True + logging.debug('CLICommand upgraded flag set') + + @property + def upgraded(self): + return self._upgraded + + def set_adv_upgraded(self): + """ Flags that this advanced command was upgraded. """ + self._adv_upgraded = True + logging.debug('CLICommand adv_upgraded flag set') + + @property + def adv_upgraded(self): + return self._adv_upgraded + + def set_invalid(self): + """ Flags that this command is invalid. """ + self._invalid = True + logging.debug('CLICommand invalid flag set') + + @property + def invalid(self): + return self._invalid + + def add_positional(self, positional_param): + """ Adds a positional parameter at the end of the parameters. + positional_param - the node containing the value of the parameter + """ + assert isinstance(positional_param, CLIPositionalParameter) + self._positionals.append(positional_param) + logging.debug('CLICommand positional parameter added: ' + + str(positional_param)) + + def add_positional_list(self, positional_params): + """ Adds a list of positional parameters. + positional_params - a list of nodes containing the parameter values + """ + for pos in positional_params: + assert isinstance(pos, CLIPositionalParameter) + self._positionals.append(pos) + logging.debug('CLICommand positional parameter added: ' + + str(pos)) + + def remove_positional(self, inx): + """ Removes the given positional parameter. + NOTE: once a positional parameter is removed the following + positional parameters' indexes are decremented by 1. + inx - the (zero-based) index of the positional parameter. + """ + assert inx < len(self._positionals) and inx >= 0 + del self._positionals[inx] + self._upgraded = True + logging.debug('CLICommand positional parameter removed at index: ' + + str(inx)) + + def add_keyword(self, keyword_param): + """ Adds a keyword parameter at the end of the parameters. + keyword_param - the keyword parameter node to add + """ + assert isinstance(keyword_param, CLIKeywordParameter) + self._keywords[keyword_param.name.name] = keyword_param + logging.debug('CLICommand keyword parameter added: ' + + str(keyword_param)) + + def add_keyword_list(self, keyword_params): + """ Adds a list of keyword parameters. + keyword_params - a list of keyword parameter nodes to add + """ + for kw in keyword_params: + assert isinstance(kw, CLIKeywordParameter) + self._keywords[kw.name.name] = kw + logging.debug('CLICommand keyword parameter added: ' + + str(kw)) + + def remove_keyword(self, name): + """ Removes the given keyword parameter. + name - the name of the keyword (without the "-") + """ + assert name in self._keywords + del self._keywords[name] + self._upgraded = True + logging.debug('CLICommand keyword parameter removed with key: ' + + str(name)) + + def remove_keyword_value(self, name, inx): + """ Some keywords will have multiple values. + This method removes a particular keyword value given by index + inx in given keyword. + name - the name of the keyword (without the "-") + inx - the (zero-based) index of the keyword value. + NOTE: once a keyword value is removed the following keyword values + indexes are decremented by 1. + """ + assert name in self._keywords + keyword_param = self._keywords[name] + assert inx < len(keyword_param.values) and inx >= 0 + del keyword_param.values[inx] + self._upgraded = True + logging.debug('CLICommand keyword parameter value removed at index: ' + + str(inx) + " with key: " + str(name)) + + def keyword_exists(self, name): + """ Determine whether the given keyword existins for this command. + name - the name of the keyword (without the "-") + Returns true iff the keyword exists for this command. + """ + return name in self._keywords + + def keyword_parameter(self, name): + """ Gets the keyword parameter for the given keyword. + name - the name of the keyword (without the "-") + Returns the parameter or None if the keyword does not exist in this + command. + """ + return self._keywords.get(name) + + def keyword_value(self, name): + """ Gets the keyword value for the given keyword. + name - the name of the keyword (without the "-") + Returns the value or None if the keyword does not exist in this + command. + """ + result = self._keywords.get(name) + if result is not None: + result = result.values + return result + + def positional_value(self, inx): + """ Gets the given positional parameter. + inx - the (zero-based) index of the positional parameter. + Returns the value or None if no such positional parameter exists. + """ + result = None + if inx < len(self._positionals) and inx >= 0: + result = self._positionals[inx] + return result + + def get_number_of_params(self): + """ Gets the number of parameters. """ + no_of_params = len(self._positionals) + len(self._keywords) + return no_of_params + + def __str__(self): + """ Creates a readable string representation of the CLI node. + Returns the string representation. + """ + if not (self._upgraded or self._adv_upgraded): + return self._original_line + else: + result = self._op + " " + self._group + " " + self._ot + for node in self._positionals: + result += " " + str(node) + for node in self._keywords.values(): + result += " " + str(node) + return result + "\n" + + def __repr__(self): + """ Creates an unambiguous representation of the CLI node. + Returns the string representation. + """ + return common.class_repr(self) + + +class CLIPositionalParameter(CLIParseTreeNode): + """ A CLI positional parameter """ + + def __init__(self, value): + """ Creates a positional node. + value - the value of the parameter + """ + self._value = value + self._quoted = False + super(CLIPositionalParameter, self).__init__() + logging.debug('CLIPositionalParameter created: value=' + str(value)) + + @property + def value(self): + return self._value + + @property + def quoted(self): + return self._quoted + + @quoted.setter + def quoted(self, quoted): + self._quoted = quoted + + def set_value(self, value, quoted=False): + """ Set the value of this parameter. + value - the value to set + quoted - true if the value is in properly quoted output format + """ + self._value = value + self._quoted = quoted + logging.debug('CLIPositionalValue value updated: value=' + str(value) + + ', quoted=' + str(quoted)) + + def __str__(self): + """ Creates a readable string representation of the positional + parameter. + Returns the string representation. + """ + if self._quoted: + return self._value + else: + return self.normalize(self._value) + + def __repr__(self): + """ Creates an unambiguous representation of the positional parameter. + Returns the string representation. + """ + return common.class_repr(self) + + +class CLIKeywordParameter(CLIParseTreeNode): + """ A CLI keyword parameter """ + + def __init__(self, name): + """ Creates a keyword parameter, initialized to have no value. + name - the keyword name node + """ + assert isinstance(name, CLIKeywordName) + self._name = name + self._values = [] + super(CLIKeywordParameter, self).__init__() + logging.debug('CLIKeywordParameter created: name=' + str(name)) + + def add_value(self, value): + """ Adds a value to the end of the list of keyword values. + value - the keyword value node to add + """ + child = CLIKeywordValue(value) + self._values.append(child) + logging.debug('CLIKeywordParameter value added: ' + str(value)) + + def add_value_list(self, values): + """ Adds a list of keyword values. + values - a list of keyword values + """ + for val in values: + self.add_value(val) + + def __str__(self): + """ Creates a readable string representation of the keyword parameter + node. + Returns the string representation. + """ + result = str(self._name) + for value in self._values: + result += " " + str(value) + return result + + def __repr__(self): + """ Creates an unambiguous representation of the keyword parameter. + Returns the string representation. + """ + return common.class_repr(self) + + @property + def name(self): + """ Gets the keyword name node. + Returns the keyword name node. + """ + return self._name + + @property + def values(self): + """ Gets the keyword value nodes. + Returns the list of keyword value nodes. + """ + return self._values + + +class CLIKeywordName(CLIParseTreeNode): + """ A CLI keyword name """ + + def __init__(self, name): + """ Creates a keyword name node. + name - the name of the keyword (without the "-") + """ + self._name = name + super(CLIKeywordName, self).__init__() + logging.debug('CLIKeywordName created: name=' + name) + + @property + def name(self): + """ Gets the keyword name. + Returns the keyword name. + """ + return self._name + + def __str__(self): + """ Creates a readable string representation of the keyword name node. + Returns the string representation. + """ + return "-" + self._name + + def __repr__(self): + """ Creates an unambiguous representation of the keyword name node. + Returns the string representation. + """ + return common.class_repr(self) + + +class CLIKeywordValue(CLIParseTreeNode): + """ The value for a CLI keyword parameter """ + + def __init__(self, value): + """ Creates a keyword value node. + value - a value for the keyword. + """ + self._value = value + self._quoted = False + super(CLIKeywordValue, self).__init__() + logging.debug('CLIKeywordValue created: value=' + str(value)) + + @property + def value(self): + return self._value + + @property + def quoted(self): + return self._quoted + + @quoted.setter + def quoted(self, quoted): + self._quoted = quoted + + def set_value(self, value, quoted=False): + """ Set the value of this parameter. + value - the value to set + quoted - true if the value is in properly quoted output format + """ + self._value = value + self._quoted = quoted + logging.debug('CLIKeywordValue value updated: value=' + str(value) + + ', quoted=' + str(quoted)) + + def __str__(self): + """ Creates a readable string representation of the keyword value node. + Returns the string representation. + """ + if self._quoted: + return self._value + else: + return self.normalize(self._value) + + def __repr__(self): + """ Creates an unambiguous representation of the keyword value node. + Returns the string representation. + """ + return common.class_repr(self) diff --git a/nspepi/nspepi2/pi_lex.py b/nspepi/nspepi2/pi_lex.py new file mode 100644 index 0000000..bc98eb1 --- /dev/null +++ b/nspepi/nspepi2/pi_lex.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python2 + +# Copyright 2021 Citrix Systems, Inc. All rights reserved. +# Use of this software is governed by the license terms, if any, +# which accompany or are included with this software. + +import logging +import re + + +class PILex(object): + """ + Class to parse PI expressions. + """ + + @staticmethod + def get_pi_string(expr): + """ + Helper function to get classic expression from + SYS.EVAL_CLASSIC_EXPR("<>"). + expr - should be substring which starts from opening quote in + SYS.EVAL_CLASSIC_EXPR expression to the end of string. + Example: + "ns_true") && true - Returns ns_true + Return values: + -classic expression after removing quotes and handling backslashes + -length of classic expression including double quotes in original + expression expr. + """ + if not expr.startswith('"'): + return None + index = 0 + value = "" + # Increment by 1 for opening quote + index += 1 + expr_length = len(expr) + while index < expr_length: + if expr[index] == '\\': + index += 1 + if index >= expr_length: + return None + if expr[index] in '\\\'"': + value += expr[index] + elif expr[index] == 't': + value += '\t' + elif expr[index] == 'r': + value += '\r' + elif expr[index] == 'n': + value += '\n' + elif expr[index] == 'x': + # Taking next 2 characters to validate for hex digits and + # then to convert to byte. + # Now index points to 2nd hex digit + index += 2 + if (index < expr_length and re.match(r"^[0-9a-fA-F]{2}$", + expr[index - 1: index + 1])): + hex_digits = expr[index - 1: index + 1] + hex_digits = int(hex_digits, 16) + if hex_digits > 127: + logging.error("Invalid hex value is used. Maximum " + "hex value allowed is 7f.") + return None + value += chr(hex_digits) + else: + return None + elif expr[index] in "01234567": + # Check for oct digits and convert to byte. + m = re.match(r"^([0-7]{1,3})", expr[index:]) + oct_digits = m.group(1) + oct_digits_length = len(oct_digits) + # Now index points to last octal digit. + index += oct_digits_length - 1 + oct_digits = int(oct_digits, 8) + if oct_digits > 127: + logging.error("Invalid octal value is used. Maximum " + "octal value allowed is 177.") + return None + value += chr(oct_digits) + else: + return None + elif expr[index] == '"': + break + else: + value = value + expr[index] + index += 1 + if index >= expr_length: + return None + # Increment by 1 for closing quote. + value_length = index + 1 + return [value, value_length] diff --git a/nspepi/validation-conversion-script.md b/nspepi/validation-conversion-script.md new file mode 100644 index 0000000..0dc8086 --- /dev/null +++ b/nspepi/validation-conversion-script.md @@ -0,0 +1,188 @@ +# Scripts for pre-validating and converting deprecated features + +In Citrix ADC release version 13.1, some features and functionalities have been removed. If the current ADC configuration contains any of those features and functionalities, then that configuration would be lost after upgrading to 13.1. Before upgrading to 13.1, you must first convert the configuration to the Citrix recommended alternatives to avoid such loss of configurations. Citrix provides scripts to help you with the conversion. + +The following features and functionalities have been removed in 13.1: + +- Filter feature. +- Speedy (SPDY), sure connect (SC), priority queuing (PQ), HTTP denial of service (DoS), and HTML injection features. +- Classic policies for content switching, Cache redirection, compression, and application firewall. +- URL and domain parameters in the `add cs policy` command. +- Classic expressions in load balancing persistence rules. +- The `pattern` parameter in Rewrite actions. +- The `bypassSafetyCheck` parameter in the Rewrite actions. +- `SYS.EVAL_CLASSIC_EXPR` in advanced expressions. +- The `policy patClass` configuration entity. +- `HTTP.REQ.BODY` with no argument in advanced expressions. +- Q and S prefixes in the advanced expressions. +- The `policyType` parameter for the `cmp parameter `setting. + +The following tools help with the conversion: + +- Validation tool for detecting removed deprecated features and functionalities in Citrix ADC version 13.1. +- NSPEPI tool for converting deprecated commands/features to non-deprecated commands/features. + +For using the conversion tools, copy the files from here to your Citrix ADC appliance as per the instructions: + +1. Clone the repo `https://github.com/citrix/ADC-scripts.git` and goto `ADC-scripts/nspepi` directory. +2. Copy `nspepi` and `check_invalid_config` files to the `/netscaler` path in Citrix ADC. +3. Copy all files under the `nspepi2` directory to the `/netscaler/nspepi2` path in Citrix ADC. + +## Pre-validation tool for removed or deprecated features in Citrix ADC version 13.1 + +This is a pre-validation tool to check if any deprecated functionality that is removed from Citrix ADC 13.1 is still used in the configuration for your current release. If the validation result shows usage of a deprecated functionality, then before upgrading your appliance, you must first modify the configuration to the Citrix recommended alternative. You can modify the configuration either manually or using the NSPEPI tool. + +### Running the validation tool: + +This tool needs to be run from the command line of the shell within the Citrix ADC appliance (you need to type the `shell` command on the Citrix ADC CLI). + + check_invalid_config + + +The `config_file` parameter: The configuration file that needs to be checked and it should be from a saved configuration, such as in the `ns.conf` file. + +### Examples: + +The following example shows when the configuration file contains deprecated functionality that is removed from Citrix ADC 13.1. + + # check_invalid_config /nsconfig/ns.conf + + The following configuration lines get errors in 13.1 and these configurations and also any dependent configuration is removed from the configuration: + add policy expression x "sys.eval_classic_expr(\"ns_true\")" + add cmp policy cmp_pol -rule ns_true -resAction GZIP + add cs policy cs_pol_2 -rule ns_trueadd cs policy cs_pol_3 -domain www.abc.com + add cs policy cs_pol_4 -url "/abc" + add rewrite action act_1 replace_all "http.req.body(1000)" http.req.url -pattern abcd + add rewrite action act_123 replace_all http.req.url "\"aaaa\"" -pattern abcd + add responder action ract respondwith "Q.URL + Q.HEADER(\"abcd\")" + add responder policy rsp_pol "sys.eval_classic_expr(\"ns_true\")" DROP + add appfw policy aff_pol_1 "http.req.body.length.gt(10)" APPFW_BYPASS + add appfw policy aff_pol ns_true APPFW_BYPASS + + The nspepi upgrade tool can be useful in converting your configuration. For more information, see the documentation at https://docs.citrix.com/en-us/citrix-adc/current-release/appexpert/policies-and-expressions/introduction-to-policies-and-exp/converting-policy-expressions-nspepi-tool.html. + + +The following is an example when the configuration file does not contain any deprecated functionality that is removed from Citrix ADC 13.1. + + # check_invalid_config /var/tmp/new_ns.conf + + No issue detected with the configuration. + +## NSPEPI tool + +The `NSPEPI` tool helps in converting the deprecated commands or features to the Citrix recommended alternatives. + +## Running the NSPEPI tool + +This tool needs to be run from the command line of the shell (you should type the `shell` command on the Citrix ADC CLI). + + nspepi [-h] (-e | -f ) [-d] [-v] [-V] + +Parameters: + +- -h, --help: shows help message and exit +- -e ,--expression : converts classic policy expression to advanced policy expression (maximum length of 8191 allowed) +- -f , --infile : converts Citrix ADC configuration file +- -d, --debug: log debug output +- -v, --verbose: shows verbose output +- -V, --version: shows the version number of the program and exit + +**Note:** Either the `-f` or `-e` parameter must be specified to perform a conversion. Use of the `-d` parameter is intended for the Citrix support team to analyze for support purposes. + +The NSPEPI tool does not modify the input file. Instead, it generates two files with prefixes `new_` and `warn_` and they are put into the same directory as where the input configuration file is present. The file with the `new_ prefix` contains the converted configuration. And the file with `warn_ prefix` contains the warnings and errors. If there are any warnings or errors that got generated in the warn file, the errors must be fixed manually as part of the conversion process. Once converted, you must test the file in a test environment and then use it in the production environment to replace the actual `ns.conf` config file. After testing, you must reboot the appliance using the newly converted `ns.conf` config file. + +Following are a few examples of running the NSPEPI tool from the command line interface: + +Example output for –e parameter: + + # nspepi -e "req.http.header foo == \"bar\"" + "HTTP.REQ.HEADER(\"foo\").EQ(\"bar\")" + +Example output for -f parameter: + +- Example when there are no warnings or errors: + + # cat sample.conf + add cr vserver cr_vs HTTP -cacheType TRANSPARENT -cltTimeout 180 -originUSIP OFF + add cr policy cr_pol1 -rule ns_true + bind cr vserver cr_vs -policyName cr_pol1 + + # nspepi -f sample.conf + + Converted config will be available in a new file new_sample.conf. + Check warn_sample.conf file for any warnings or errors that might have been generated. + + # cat new_sample.conf + + add cr vserver cr_vs HTTP -cacheType TRANSPARENT -cltTimeout 180 -originUSIP OFF + add cr policy cr_pol1 -rule TRUE -action ORIGIN + bind cr vserver cr_vs -policyName cr_pol1 -priority 100 -gotoPriorityExpression END -type REQUEST + + # cat warn_sample.conf + # + +- Example when there are warnings or errors: + + # cat sample_2.conf + add policy expression security_expr "req.tcp.destport == 80" -clientSecurityMessage "Not allowed" + set cmp parameter -policyType CLASSIC + add cmp policy cmp_pol1 -rule ns_true -resAction COMPRESS + add cmp policy cmp_pol2 -rule ns_true -resAction COMPRESS + add cmp policy cmp_pol3 -rule TRUE -resAction COMPRESS + bind cmp global cmp_pol1bind cmp global cmp_pol2 -state DISABLED + bind cmp global cmp_pol3 -priority 1 -gotoPriorityExpression END -type RES_DEFAULT + bind lb vserver lb_vs -policyName cmp_pol2 + + # nspepi –f sample_2.conf + + Converted config will be available in a new file new_sample_2.conf. + Check warn_sample_2.conf file for any warnings or errors that might have been generated. + + # cat new_sample_2.conf + add policy expression security_expr "req.tcp.destport == 80" -clientSecurityMessage "Not allowed" + add cmp policy cmp_pol1 -rule TRUE -resAction COMPRESSadd cmp policy cmp_pol2 -rule TRUE -resAction COMPRESS + add cmp policy cmp_pol3 -rule TRUE -resAction COMPRESS + # bind cmp global cmp_pol2 -state DISABLED#bind cmp global cmp_pol3 -priority 1 -gotoPriorityExpression END -type RES_DEFAULT + bind cmp global cmp_pol1 -priority 100 -gotoPriorityExpression END -type RES_DEFAULT + bind lb vserver lb_vs -policyName cmp_pol2 -priority 100 -gotoPriorityExpression END -type RESPONSE + + # cat warn_sample_2.conf + + 2021-08-15 23:38:04,337: ERROR - Error in converting expression security_expr : conversion of clientSecurityMessage based expression is not supported. + + 2021-08-15 23:38:05,136: WARNING - Following bind command is commented out because state is disabled. If command is required please take a backup because comments will not be saved in ns.conf after triggering 'save ns config': bind cmp global cmp_pol2 -state DISABLED + + 2021-08-15 23:38:05,138: WARNING - Bindings of advanced CMP policies to cmp global are commented out, because initial global cmp parameter isclassic but advanced policies are bound. Now global cmp parameter policytype is set to advanced. If commands are required please take a backup because comments will not be saved in ns.conf after triggering 'save ns config'. + + + +- Example output of the -f parameter along with -v parameter + + # nspepi -f sample.conf -v + INFO - add cr vserver cr_vs HTTP -cacheType TRANSPARENT -cltTimeout 180 -originUSIP OFF + INFO - add cr policy cr_pol1 -rule TRUE -action ORIGIN + INFO - bind cr vserver cr_vs -policyName cr_pol1 -priority 100 -gotoPriorityExpression END -type REQUEST + + Converted config will be available in a new file new_sample.conf. + Check warn_sample.conf file for any warnings or errors that might have been generated. + +### Commands or features handled by the NSPEPI conversion tool + + - The following classic policies are converted to advanced policies. These policies include conversion of entity bindings including global bindings. + - add appfw policy + - add cmp policy + - add cr policy + - add cs policy + - add tm sessionPolicy + - add tunnel trafficPolicy + + - The rule parameter configured in `add lb vserver` is converted from classic expression to advanced expression. + - Filter feature (except the FORWARD type filter action) + - Named expressions (`add policy expression` commands). Each classic named policy expression is converted to its corresponding advanced named expression with `nspepi_adv_` set as the prefix. In addition, usage of named expressions for the converted classic expressions is changed to the corresponding advanced named expressions. Also, every named expression has two named expressions, where one is classic and the other one is advanced. + - The SPDY parameter configured in `add ns httpProfile` or `set ns httpProfile` command is changed to `-http2 ENABLED`. + - Patclass feature + - Pattern parameter in rewrite action + - SYS.EVAL_CLASSIC_EXPR is converted to the equivalent non-deprecated advanced expression. These expressions can be seen in any command where advanced expressions are allowed. + - Q and S prefixes of advanced expressions are converted to equivalent non-deprecated advanced expressions. These expressions can be seen in any command where advanced expressions are allowed. + +For more information on the NSPEPI tool, see the [Citrix ADC documentation](https://docs.citrix.com/en-us/citrix-adc/current-release/appexpert/policies-and-expressions/introduction-to-policies-and-exp/converting-policy-expressions-nspepi-tool.html). diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index c91137e..0000000 --- a/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -mkdocs>=1 -pygments>=2.2 -pymdown-extensions>=4.3 \ No newline at end of file diff --git a/runtime.txt b/runtime.txt deleted file mode 100644 index 56e972a..0000000 --- a/runtime.txt +++ /dev/null @@ -1 +0,0 @@ -3.5 \ No newline at end of file diff --git a/td-to-ap/migration-script-td.md b/td-to-ap/migration-script-td.md new file mode 100644 index 0000000..5bc9478 --- /dev/null +++ b/td-to-ap/migration-script-td.md @@ -0,0 +1,202 @@ +# Migrating traffic domain configuration on a Citrix ADC to admin partition configuration using the tdToPartition.pl script + +Citrix ADC provides different options to create multiple isolated environments within a same Citrix ADC appliance such as traffic domains and admin partitions. Admin partitions provide much better control and management than traffic domains. + +You can migrate the traffic domain configuration on a Citrix ADC to the admin partition configuration using the `tdToPartition.pl` script provided by Citrix. + + +## Running the script + +Download the `tdToPartition.pl` using below command: + + wget https://raw.githubusercontent.com/citrix/ADC-scripts/master/td-to-ap/tdToPartition.pl + + +The following commands show the usage of the script. + #./tdToPartition.pl + + perl tdToPartition.pl + +The parameters you need to provide are explained as follows: + +- `TD-PartName_mapping_file` - This file is the traffic domain to admin partition name mapping input file that needs to be done by the customer. + + +- `input_config_file` - This file is the existing traffic domain deployment configuration input file (`ns.conf`) for Citrix ADC + +- `output_config_file` - This file is the newly generated output file (`td_migration_ap.conf`) after running the migration script on input files (`ns.conf` and `TD-PartName_mapping_file`). + + +The following example shows the content of a sample `TD-PartName_mapping_file` named `map.csv`. + + # cat map.csv + 256|ap_256 + 1150|ap_1150 + 206|ap_206 + 247|ap_247 + 253|ap_253 + 32|ap_32 + 702|ap_702 + +The contents of the files are in the `TD-name` | `admin-partition-name` format. + + +During the run of the migration tool script, corresponding to each admin partition `ns.conf` file, the respective log file is generated. You can apply this newly generated output file (`td_migration_ap.conf`) using the ``batch -fileName `` command from the Citrix ADC CLI. + +**Note** +You must verify and ensure that there is no loss of configuration during the migration. + + +Following is the content of a sample `td_migration_ap.conf` file. + + # cat td_migration_ap.conf + + rm trafficDomain 1 + add ns partition Partition-SECURE + bind ns partition Partition-SECURE -vlan 40 + switch partition Partition-SECURE + batch -filename /var/td_migration_ap.conf.Partition-SECURE.conf -outfile /var/td_migration_ap.conf.Partition-SECURE.out + save config + switch partition DEFAULT + save config + + + +## Specific deployment scenarios and changes + +Based on how the configured VLANs are associated to traffic domains in the existing deployment, the following two scenarios need to be addressed while migrating from traffic domains to admin partitions. + +- Tagged VLANs bound to a traffic domain: This deployment works as per the steps mentioned in stand-alone and HA deployment scenarios. + +- Untagged VLANs bound to a traffic domain: For this scenario, you need to consider the following two options depending upon their existing deployment and need to consider the following explicit configuration before applying the configuration generated from the migration script so that the partition VLAN binding does not fail. + + - IP addresses are overlapping: You need to change the untagged VLAN configuration to the tagged VLAN configuration. IP address overlapping is supported with dedicated VLANs (tagged VLANs) for admin partitions. + + - IP addresses are non-overlapping or existing untagged VLAN cannot be configured as tagged: + You need to make the untagged VLAN as shared VLAN because only the untagged shared VLAN can be bound to the admin partition since the introduction of the shared VLAN feature for admin partition. From Citrix ADC release version 11.1, IP address overlapping is not supported with shared VLAN for admin partitions because the shared VLAN can be bound for more than one partition. + +### Untagged VLANs: overlapping IP addresses deployment scenario + +In this scenario, the traffic domain configuration contains untagged VLANs and it is converted to the admin partition configuration. + +Following is a sample traffic domain configuration for migration. + + add vlan 102 + bind vlan 101 -ifnum 1/2 + bind vlan 102 -ifnum 1/3 + bind ns trafficDomain 101 -vlan 101 + bind ns trafficDomain 102 -vlan 102 + + + To avoid errors, you must change the configuration in the `ns.conf` file before running the migration script: + + bind vlan 101 -ifnum 1/2 -tagged + bind vlan 101 -ifnum 1/3 -tagged + +After running the migration script on the `ns.conf` file, the following is the admin partition configuration: + + + rm trafficDomain 102 + rm trafficDomain 101 + add ns partition "INSIDE2" + add ns partition "INSIDE" + bind ns partition "INSIDE" -vlan 101 + bind ns partition "INSIDE2" -vlan 102 + + +You may see the following errors if you do not change the configuration in the `ns.conf` file. + + > bind ns partition "INSIDE" -vlan 101 + ERROR: The specified VLAN cannot be bound to a partition because it is configured as untagged member of interface. + + > bind ns partition "INSIDE2" -vlan 102 + ERROR: The specified VLAN cannot be bound to a partition because it is configured as untagged member of interface. + +### Untagged VLANs: non-overlapping IP addresses deployment scenario + +Following is a sample traffic domain configuration for migration. + + add ns trafficDomain 101 + add ns trafficDomain 102 + add vlan 101 + add vlan 102 + bind vlan 101 -ifnum 1/2 + bind vlan 102 -ifnum 1/3 + bind ns trafficDomain 101 -vlan 101 + bind ns trafficDomain 102 -vlan 102 + +You should add the following configuration in the migration configuration file before running the migration script: + + set vlan 101 -sharing ENABLED + set vlan 102 -sharing ENABLED + +After running the migration script on the `ns.conf` configuration, the following is the admin partition configuration: + + + rm trafficDomain 102 + rm trafficDomain 101 + add ns partition "INSIDE2" + add ns partition "INSIDE" + bind ns partition "INSIDE" -vlan 101 + bind ns partition "INSIDE2" -vlan 102 + + + +You may see the following errors if the migration configuration file is not modified before running the migration script: + + + > bind ns partition "INSIDE" -vlan 101 + ERROR: The specified VLAN cannot be bound to a partition because it is configured as untagged member of interface. + > bind ns partition "INSIDE2" -vlan 102 + ERROR: The specified VLAN cannot be bound to a partition because it is configured as untagged member of interface. + + +### Cross-traffic domain binding configurations + +If you are using cross traffic domain binding or referencing across traffic domains, it is allowed only for virtual server service binding (that means `vserver` in the default traffic domain and service in another traffic domain). Before applying the migration script on the existing `ns.conf` file, you should take care of the cross traffic domain binding. + +**Note:** Cross traffic domain configuration is not allowed across admin partitions. + +## Detailed migration steps + +Following are the migration steps for stand-alone and high availability deployments. + +### Stand-alone deployment + +For stand-alone deployments, here are the details of migration steps: + +1. Create the admin partition configuration first. You can use the migration script, which can help to pull the TD configuration and dependencies into a single file. + +1. Add the admin partition. +1. If the admin partition configuration refers any SSL certificates, then copy them to the `/var/partitions//ssl` folder. +1. Batch the configuration inside the admin partition. When you perform this step, VLAN-specific configurations such as IP bindings may fail. +1. Verify if any configuration failed due to a configuration not being present in the admin partition. Then you need to include the missing configuration and retry. +1. Now, unbind all VLANs from the traffic domain and bind them to the admin partition. Apply the VLAN specific configuration inside the admin partition. + +1. Check that all services are coming up properly. + +1. If any problem is seen, unbind the VLANs from the admin partition, bind them back to the traffic domain so that traffic domain configuration can become active again. + +1. Otherwise, remove the traffic-domain. Save the configuration inside the admin partition. + +### High availability + +For high availability deployments, here are the details of migration steps: + +1. Create the admin partition configuration first. You can use the migration script, which can help to pull the traffic domain configuration and dependencies in to a single file. + +1. Disable high availability synchronization and propagation on both primary and secondary Citrix ADCs. (Use the `set ha node -hasync disABLED -haprop disabled` command.) +1. In the secondary Citrix ADC, remove the traffic domain. +1. Add the admin partition. +1. Bind to the admin partition. +1. If the admin partition configuration refers any SSL certificates, then copy them to the `/var/partitions//ssl` folder. +1. Batch the configuration inside the admin partition. +1. Verify that if any configuration was failed due to a configuration not being present in the admin partition. Then you need to include the missing configuration and retry. +1. Perform a forced failover. +1. Verify that all services are coming up properly. +1. If there are any issues, perform the forced failover again. +1. Otherwise, enable high availability synchronization and propagation. + +**Note:** Once the admin partitions are up, 10 MB is configured as the default memory for each partition. The memory of partitions can be changed according to your requirements. + + diff --git a/td-to-ap/tdToPartition.pl b/td-to-ap/tdToPartition.pl new file mode 100644 index 0000000..21d47e6 --- /dev/null +++ b/td-to-ap/tdToPartition.pl @@ -0,0 +1,303 @@ +# Copyright 2021 Citrix Systems, Inc. All rights reserved. +# Use of this software is governed by the license terms, if any, +# which accompany or are included with this software. +#!/usr/bin/perl -w +use strict; +use warnings; + +use List::Util qw(first); +use Text::ParseWords; +use Text::Balanced qw(extract_bracketed); +use Cwd 'abs_path'; +use Data::Dumper; +use File::Basename; +our $PERL_SINGLE_QUOTE; + +# Expect at least three argument from CLI. +if (($#ARGV+1) < 3) +{ + print "Usage: perl tdToPartition.pl \n"; + exit; +} + +# this will validate the string and decides whether to add it to out hash table. + +#no tokenizing. only command will be copied. +my $cmds_blind_copy = qr/enable ns feature|enable ns mode/; +#these commands are ignored from copying. add audit messageaction is not supported in partition. +my $cmds_blind_ignore = qr/add audit messageaction|bind system global/; +# copies the command and also will capture its dependents later. +my $cmds_copy_dependents = qr/bind \S+ global|add policy patset/; + + +sub is_string_valid +{ + my $str = shift(@_); + + if ($str eq "") + { + #print "$str invalid-1\n"; + return 0; + } + + if ($str =~ m/[^a-zA-Z0-9\_\#\.\:\-\=\@]/){ + #print "$str invalid-2\n"; + return 0; + } + if ( $str =~ /[^0-9]/) + { + ; + } + else + { + #print "$str invalid-3\n"; + return 0; + } + if ($str =~m/ENABLED|DISABLED|ON|OFF|^-\S+|UP|DOWN|END|REQUEST|YES/) + { + #print "$str invalid-4\n"; + return 0; + } + if( $str=~ m/^(\d\d?\d?)\.(\d\d?\d?)\.(\d\d?\d?)\.(\d\d?\d?)/ && + ( $1 <= 255 && $2 <= 255 && $3 <= 255 && $4 <= 255 )) + { + #print "$str invalid-5\n"; + return 0; + } + return 1; +} + + +my %token_hash; +my %td_info; +my %partition_info; + +open(my $MAPPING, '<', $ARGV[0]) or die "Error: $!"; # Open input mapping file. + +while (<$MAPPING>) +{ + my $line = chomp($_); + if ($line =~m/^#/){ + ;#comment only. ignore + } + else { + my ($td, $part, $netprofile) = parse_line('\|', 1, $_); + $td_info{$td}{"pname"} = $part; + if (defined $netprofile){ + $td_info{$td}{"partition_netprofile_for_appflow"} = $netprofile; + } + } +} +close($MAPPING); + +open(my $FOUT, '>', $ARGV[2]) or die "Error: Cannot create file. $!"; # Open output config file. + +open(my $FIN, '<', $ARGV[1]) or die "Error: $!"; # Open input config file. +my @lines = <$FIN>; +close($FIN); +foreach my $line (@lines) +{ + chomp($line); +} + + +my %Cmds_3args=("aaa","","cmp","","subscriber","","ica","","rdp","","responder","","rewrite","","system","","appflow","","cr","","appfw","","appqoe","","cs","","tm","","db","","ipsec","","transform","","audit","","ns","","authentication","","sc","","dns","","tunnel","","authorization","","dos","","autoscale","","lb","","ntp","","ca","","cache","","feo","","vpn","","filter","","smpp","","snmp","","policy","","gslb","","spillover","","wf","","HA","","wi","","pq","","ssl","","lsn","","cluster","","stream",""); + + + +my @config_in_partition; + +foreach my $td (keys %td_info) +{ + my $pName = $td_info{$td}{"pname"}; + + open(my $LOG, '>', "$ARGV[2].$pName.log") or die "Error: Cannot create log file."; # Open output config file. + + open(my $PartFile, '>', "$ARGV[2].$pName.conf") or die "Error: Cannot create file. $!"; # Create temporary config file for each partition. + + @config_in_partition = (); + undef %token_hash; + + my $lines_added = 1; + my $lookup = ""; + my $token = ""; + my $line_number = 0; + my $log_str = ""; +# Parse input config file. + while ($lines_added) + { + $line_number = 0; + $lines_added = 0; + foreach my $line (@lines) + { + if (exists $config_in_partition[$line_number] && ($config_in_partition[$line_number] > 0)){ + ; + } + elsif($line =~ $cmds_blind_ignore) { + $config_in_partition[$line_number] = 2; # not related command. + } + elsif ($line =~m/-td\s(\d+)/){ + if ($td != $1) { + $config_in_partition[$line_number] = 2; # not related command. + }else { + # we need to add this line to our config. + $config_in_partition[$line_number] = 1; + $lines_added++; + my @words = split(/ /, $line); + if (scalar(@words) < 4){ + $lookup = ""; + } + elsif (exists $Cmds_3args{$words[1]}) { + $lookup = $words[3]; + $log_str = join(" ", @words[0..3]); + splice(@words, 0, 3); + }else { + $lookup = $words[2]; + $log_str = join(" ", @words[0..2]); + splice(@words, 0, 2); + } + if (is_string_valid($lookup)) { + print $LOG "$log_str <-- Marked for TD $td\n"; + foreach $token (@words){ + if (is_string_valid($token)) + { + $token_hash{$token} = $log_str; + } + } + } + else { + print $LOG "$line <-- Marked for TD $td\n"; + } + } + } + elsif ($line =~ $cmds_blind_copy){ + $config_in_partition[$line_number] = 1; + print $LOG "$log_str <-- RE is true $cmds_blind_copy\n"; + } + elsif ($line =~/bind lb group/) + { + my @words = split(/ /, $line); + my $group_name = $words[3]; + if (exists($token_hash{$words[4]})){ + $config_in_partition[$line_number] = 1; + $lines_added++; + print $LOG "$line <-- Referred by $token_hash{$words[4]}\n"; + $token_hash{$words[3]} = $line; + } + } + else + { + my @words = split(/ /, $line); + + if (scalar(@words) < 4){ + $lookup = ""; + } + elsif (exists $Cmds_3args{$words[1]}) { + $log_str = join(" ", @words[0..3]); + $lookup = $words[3]; + }else { + $log_str = join(" ", @words[0..2]); + $lookup = $words[2]; + } + if (is_string_valid($lookup)) { + if ((exists $token_hash{$lookup}) or ($line =~ $cmds_copy_dependents)) + { + $config_in_partition[$line_number] = 1; + $lines_added++; + if (exists $token_hash{$lookup}) + { + print $LOG "$log_str <-- $token_hash{$lookup}\n"; + } + else + { + print $LOG "$log_str <-- RE is true:$cmds_copy_dependents\n"; + } + + if ($line =~ /add ssl certKey .* -cert (\S+).* -key (\S+)/) + { + $partition_info{$pName}{"filesto_copy"}.="$1 $2 "; + } + #tokenize and add them to hash bucket for next round of lookup. + if(exists $Cmds_3args{$words[1]}){ + splice(@words, 0, 3); + } + else { + splice(@words, 0, 2); + } + foreach $token (@words){ + if (is_string_valid($token) && (not exists $token_hash{$token})) + { + $token_hash{$token} = $log_str; + } + } + } + } + else { + $config_in_partition[$line_number] = 2; # not related command. + } + } + $line_number++; + } + } + # write the config to a file now. + $line_number=0; + foreach my $line (@lines) + { + if (exists $config_in_partition[$line_number] && $config_in_partition[$line_number]==1) + { + my $cpline = $line; + $cpline =~s/(-td \d+)//; + $cpline =~s/-logAction \S+//; + if ($cpline =~m/add appflow collector/) + { + if (exists $td_info{$td}{"partition_netprofile_for_appflow"}){ + $cpline.=" -netprofile $td_info{$td}{\"partition_netprofile_for_appflow\"}"; + } + } + print $PartFile "$cpline\n"; + } + $line_number++; + } + close($PartFile); + close ($LOG); +} + +# finally last config file. we do a clear config create partitions and bind vlans apply config from each partition. +foreach my $key (keys %td_info) +{ + print $FOUT "rm trafficDomain $key\n"; +} +foreach my $key (keys %td_info) +{ + print $FOUT "add ns partition $td_info{$key}{\"pname\"}\n"; +} +#copy vlan partition bindings. +foreach my $line (@lines) +{ + if ($line =~m/bind ns trafficDomain (\d+) -vlan (\d+)/) + { + print $FOUT "bind ns partition $td_info{$1}{\"pname\"} -vlan $2\n"; + } +} + +foreach my $part_name (keys %partition_info) +{ + my @files_to_copy= split(" ", $partition_info{$part_name}{"filesto_copy"}); + foreach my $file (@files_to_copy) + { + print $FOUT "shell cp /nsconfig/ssl/$file /nsconfig/partitions/$part_name/ssl/\n"; + } +} +#print $FOUT "sync ha files\n"; +#print $FOUT "shell sleep 10\n"; + +foreach my $key (keys %td_info) +{ + print $FOUT "switch partition $td_info{$key}{\"pname\"}\n"; + print $FOUT "batch -filename /var/$ARGV[2].$td_info{$key}{\"pname\"}.conf -outfile /var/$ARGV[2].$td_info{$key}{\"pname\"}.out\n"; + print $FOUT "save config\n"; +} + +print $FOUT "switch partition DEFAULT\n"; +print $FOUT "save config\n"; +close($FOUT);