Merge pull request #180 from fabric-testbed/get_slices

Changes identified during tests

Author: kthare10

Dockerfile-auth

@@ -1,7 +1,7 @@
 FROM python:3.9.0
 MAINTAINER Komal Thareja<[email protected]>
 
-ARG HANDLERS_VER=1.2rc6
+ARG HANDLERS_VER=1.2.1
 
 RUN mkdir -p /usr/src/app
 WORKDIR /usr/src/app

authority.sh

@@ -1,3 +1,2 @@
 #!/usr/bin/env bash
-
-python3 -m fabric_cf.authority
+docker exec -i $1-am-db psql am fabric < psql.upgrade

broker.sh

@@ -1,3 +1,2 @@
 #!/usr/bin/env bash
-
-python3 -m fabric_cf.broker
+docker exec -i broker-db psql broker fabric < psql.upgrade

fabric_cf/__init__.py

@@ -1 +1 @@
-__VERSION__ = "1.2rc4"
+__VERSION__ = "1.2.1"

fabric_cf/actor/core/core/ticket.py

@@ -115,7 +115,7 @@ def get_ticket(self) -> ResourceTicket:
         """
         return self.resource_ticket
 
-    def add(self, *, concrete_set, configure: bool):
+    def add(self, *, concrete_set: ABCConcreteSet, configure: bool):
         raise TicketException("add() is not supported by Ticket")
 
     def change(self, *, concrete_set: ABCConcreteSet, configure: bool):

fabric_cf/actor/core/core/unit_set.py

@@ -96,7 +96,8 @@ def restore(self, *, plugin: ABCBasePlugin, reservation: ABCReservationMixin):
             self.logger = plugin.get_logger()
         self.reservation = reservation
 
-    def ensure_type(self, *, cset: ABCConcreteSet):
+    @staticmethod
+    def ensure_type(*, cset: ABCConcreteSet):
         """
         Validate the type of incoming concrete set
         @param cset cset

fabric_cf/actor/core/kernel/reservation_client.py

@@ -457,6 +457,9 @@ def prepare_ticket(self):
                 continue
 
             pred_state = self.redeem_predecessors.get(ID(uid=rid))
+            if not pred_state:
+                self.logger.error(f"Redeem predecessors not found {rid} for {self.get_reservation_id()}")
+                continue
             parent_res = pred_state.get_reservation()
             if parent_res is not None and \
                     ReservationStates(parent_res.get_state()) == ReservationStates.Ticketed:
@@ -580,7 +583,7 @@ def do_relinquish(self):
 
     def close(self):
         if self.state == ReservationStates.Nascent or self.state == ReservationStates.Failed:
-            self.logger.debug("Reservation in Nascent or failed state, transition to close")
+            self.logger.debug(f"Reservation in state: {self.state}, transition to {ReservationStates.Closed}")
             self.transition(prefix="close", state=ReservationStates.Closed, pending=self.pending_state)
             if self.broker is not None:
                 self.logger.debug("Triggering relinquish")

fabric_cf/actor/core/kernel/resource_set.py

@@ -50,47 +50,7 @@ class ResourceSet:
     ResourceSet with an attached ConcreteSet is "concrete." The ConcreteSet binds
     real resources (or a promise) for some or all of the abstract units in the
     set: for example, a concrete ResourceSet can represent a ticket or
-    lease. Adding or removing concrete resources does not affect the number of
-    abstract units. If there are fewer concrete units than abstract units, the
-    set has a "deficit".
-
-    An "elastic" ResourceSet may be filled at less than its full request, and it
-    may change size on extends. An actor may modify an elastic ResourceSet on an
-    active ReservationClient by calling "flex", if there is no pending operation
-    in progress, e.g., in preparation for an elastic extend. This class updates
-    the unit count to match the concrete resources on each reserve or extend (on
-    a server), or update (on a client).
-
-    Operations on the ConcreteSet through this class may drive probes and state
-    transitions on the underlying resources transferred in and out of the
-    ResourceSet (e.g., node configuration and node reboot for a COD authority, or
-    resource membership changes on a orchestrator). ConcreteSets are
-    responsible for their own synchronization: calls to ConcreteSet go through
-    pre-op "prepare" or post-op "service" methods in this class, which may block
-    and should not hold any higher-level locks. Most other operations are called
-    through Mapper or the Reservation class with the Manager lock held.
-
-    Implementation notes
-    The unit count is updated immediately to reflect additions or deletions
-    from the set. Updates to the unit count must occur only in the locked
-    methods. Configuration actions on the ConcreteSet (e.g., as resources join
-    and leave the set) must occur only in unlocked methods (e.g., "service"). A
-    tricky part is flex(), which updates abstract count to reflect a new request:
-    it is unlocked, which could race with an incoming unsolicited lease (which
-    are currently allowed), or with overlapping requests on the same set (which
-    are currently not allowed).
-    ResourceSet was conceived as supporting methods that are independent of
-    context and type of ConcreteSet. That ideal has eroded somewhat, and some key
-    fields and methods are specific to a particular context or role. Someday it
-    may be useful to break this into subclasses.
-    Currently leases are validated only with validateIncoming(). There may be
-    some additional checks to enforce.
-    No changes to ResourceData on merges. Needs thought and documentation. We
-    should remove the properties argument on ConcreteSet.change.
-    The 'null ticket corner case' (see above) is a source of complexity, and
-    should be cleaned up.
-    Calls that "reach around" ResourceSet to the concrete set are
-    discouraged/deprecated.
+    lease.
     """
     def __init__(self, *, concrete: ABCConcreteSet = None, gained: ABCConcreteSet = None,
                  lost: ABCConcreteSet = None, modified: ABCConcreteSet = None,

fabric_cf/actor/core/policy/network_node_inventory.py

@@ -89,8 +89,12 @@ def __check_capacities(self, *, rid: ID, requested_capacities: Capacities, deleg
     def __set_ips(self, *, req_ifs: InterfaceSliver, lab: Labels):
         if req_ifs.labels is not None and req_ifs.labels.ipv4 is not None:
             lab.ipv4 = req_ifs.labels.ipv4
+            if req_ifs.labels.ipv4_subnet is not None:
+                lab.ipv4_subnet = req_ifs.labels.ipv4_subnet
         if req_ifs.labels is not None and req_ifs.labels.ipv6 is not None:
             lab.ipv6 = req_ifs.labels.ipv6
+            if req_ifs.labels.ipv6_subnet is not None:
+                lab.ipv6_subnet = req_ifs.labels.ipv6_subnet
         return lab
 
     def __update_shared_nic_labels_and_capacities(self, *, available_component: ComponentSliver,

fabric_cf/actor/core/policy/network_service_inventory.py

@@ -208,6 +208,7 @@ def allocate(self, *, rid: ID, requested_ns: NetworkServiceSliver, owner_switch:
             - exclude the 1st subnet (reserved for control plane)
             - exclude the subnets already assigned to other V4/V6 NetworkService on the same owner switch
             - allocate the first available subnet to the NetworkService
+        :param rid: Reservation ID
         :param requested_ns: Requested NetworkService
         :param owner_switch: BQM Owner site switch identified to serve the NetworkService
         :param existing_reservations: Existing Reservations which also are served by the owner switch

fabric_cf/actor/core/proxies/kafka/translate.py

@@ -104,7 +104,8 @@ def translate_slice_to_avro(*, slice_obj: ABCSlice) -> SliceAvro:
         avro_slice.config_properties = slice_obj.get_config_properties()
         avro_slice.set_lease_end(lease_end=slice_obj.get_lease_end())
         avro_slice.set_lease_start(lease_start=slice_obj.get_lease_start())
-        avro_slice.set_project_id(project_id=slice_obj.get_project_id())
+        if hasattr(slice_obj, 'project_id') is not None:
+            avro_slice.set_project_id(project_id=slice_obj.get_project_id())
 
         if slice_obj.get_resource_type() is not None:
             avro_slice.set_resource_type(str(slice_obj.get_resource_type()))
@@ -123,7 +124,8 @@ def translate_auth_to_avro(*, auth: AuthToken) -> AuthAvro:
         result.guid = str(auth.get_guid())
         result.oidc_sub_claim = auth.get_oidc_sub_claim()
         result.email = auth.get_email()
-        result.token = auth.get_token()
+        if hasattr(auth, 'token') is not None:
+            result.token = auth.get_token()
         return result
 
     @staticmethod

fabric_cf/actor/core/time/actor_clock.py

@@ -185,5 +185,8 @@ def from_milliseconds(*, milli_seconds) -> datetime:
 
     @staticmethod
     def to_milliseconds(*, when: datetime) -> int:
-        epoch = datetime.fromtimestamp(0, timezone.utc)
+        if when.tzinfo is not None:
+            epoch = datetime.fromtimestamp(0, timezone.utc)
+        else:
+            epoch = datetime.fromtimestamp(0)
         return int((when - epoch).total_seconds() * 1000)

fabric_cf/actor/fim/fim_helper.py

@@ -33,7 +33,7 @@
 from fim.graph.resources.neo4j_arm import Neo4jARMGraph
 from fim.graph.resources.neo4j_cbm import Neo4jCBMGraph, Neo4jCBMFactory
 from fim.graph.slices.abc_asm import ABCASMPropertyGraph
-from fim.graph.slices.neo4j_asm import Neo4jASMFactory, Neo4jASM
+from fim.graph.slices.neo4j_asm import Neo4jASMFactory
 from fim.graph.slices.networkx_asm import NetworkxASM, NetworkXASMFactory
 from fim.slivers.attached_components import ComponentSliver
 from fim.slivers.base_sliver import BaseSliver
@@ -42,7 +42,7 @@
 from fim.slivers.interface_info import InterfaceSliver, InterfaceType
 from fim.slivers.network_node import NodeSliver
 from fim.slivers.network_service import NetworkServiceSliver
-from fim.user import ExperimentTopology, Labels, ServiceType, NodeType
+from fim.user import ExperimentTopology, Labels, NodeType
 
 
 class InterfaceSliverMapping:
@@ -288,6 +288,16 @@ def update_node(*, graph_id: str, sliver: BaseSliver):
                         node.components[cname].set_properties(label_allocations=component.label_allocations,
                                                               capacity_allocations=component.capacity_allocations,
                                                               node_map=component.node_map)
+                        # Update Mac address
+                        for ns in component.network_service_info.network_services.values():
+                            if ns.interface_info is None or ns.interface_info.interfaces is None:
+                                continue
+
+                            for ifs in ns.interface_info.interfaces.values():
+                                topo_component = node.components[cname]
+                                topo_ifs = topo_component.interfaces[ifs.get_name()]
+                                topo_ifs.set_properties(label_allocations=ifs.label_allocations)
+
             elif isinstance(sliver, NetworkServiceSliver):
                 node = neo4j_topo.network_services[node_name]
                 node.set_properties(label_allocations=sliver.label_allocations,

fabric_cf/actor/security/auth_token.py

@@ -90,7 +90,7 @@ def clone(self):
         return AuthToken(name=self.name, guid=self.guid)
 
     def __str__(self):
-        return f"name: {self.name} guid: {self.guid} email: {self.email} token: {self.token}"
+        return f"name: {self.name} guid: {self.guid} email: {self.email}"
 
     def get_token(self):
         """

fabric_cf/actor/security/fabric_token.py

@@ -118,7 +118,7 @@ def get_project_and_tags(self) -> Tuple[str or None, List[str] or None]:
             project = key
             for tag in value:
                 tag_list.append(tag)
-                break
+            break
         return project, tag_list
 
     def __str__(self):

fabric_cf/actor/security/pdp_auth.py

@@ -99,8 +99,7 @@ def _headers() -> dict:
         }
         return headers
 
-    @staticmethod
-    def build_pdp_request(*, fabric_token: FabricToken, action_id: ActionId,
+    def build_pdp_request(self, *, fabric_token: FabricToken, action_id: ActionId,
                           resource: BaseSliver or ExperimentTopology, lease_end_time: datetime) -> dict:
         """
         Build PDP Request
@@ -122,6 +121,8 @@ def build_pdp_request(*, fabric_token: FabricToken, action_id: ActionId,
         # next we need to set the owner of the resource and their projects
         # generally only the id is needed. If action is create, it's not needed at all
         project, tag_list = fabric_token.get_project_and_tags()
+        self.logger.debug("project: %s", project)
+        self.logger.debug("tag_list: %s", tag_list)
         if project is None:
             raise PdpAuthException("No project found in fabric token")
 

fabric_cf/authority/config.site.am.yaml

@@ -45,7 +45,7 @@ runtime:
   kafka-sasl-consumer-password:
   prometheus.port: 11000
   kafka.request.timeout.ms: 120000
-  rpc.request.timeout.seconds: 900
+  rpc.request.timeout.seconds: 1200
 
 logging:
   ## The directory in which actor should create log files.

fabric_cf/broker/config.broker.yaml

@@ -45,7 +45,7 @@ runtime:
   kafka-sasl-consumer-password:
   prometheus.port: 11000
   kafka.request.timeout.ms: 120000
-  rpc.request.timeout.seconds: 900
+  rpc.request.timeout.seconds: 1200
 
 logging:
   ## The directory in which actor should create log files.

fabric_cf/orchestrator/config.orchestrator.yaml

@@ -46,7 +46,7 @@ runtime:
   orchestrator.rest.port: 8700
   prometheus.port: 11000
   kafka.request.timeout.ms: 120000
-  rpc.request.timeout.seconds: 900
+  rpc.request.timeout.seconds: 1200
 
 logging:
   ## The directory in which actor should create log files.

fabric_cf/orchestrator/core/orchestrator_handler.py

@@ -374,7 +374,9 @@ def get_slices(self, *, token: str, slice_id: str = None, states: List[str] = No
 
             fabric_token = self.__authorize_request(id_token=token, action_id=ActionId.query)
 
-            project, tags = fabric_token.get_project_and_tags()
+            # TODO - uncomment once project based view is implemented on portal
+            #project, tags = fabric_token.get_project_and_tags()
+            project = None
             slice_list = controller.get_slices(slice_id=slice_guid, state=slice_states,
                                                email=fabric_token.get_email(), project=project)
             return ResponseBuilder.get_slice_summary(slice_list=slice_list)

fabric_cf/orchestrator/swagger_server/response/utils.py

@@ -23,11 +23,11 @@
 #
 #
 # Author: Komal Thareja ([email protected])
-import connexion
+from flask import request
 
 
 def get_token() -> str:
-    token = connexion.request.headers.get('Authorization', None)
+    token = request.headers.get('Authorization', None)
     if token is not None:
         token = token.replace('Bearer ', '')
-    return token
+    return token

orchestrator.sh

@@ -1,3 +1,2 @@
 #!/usr/bin/env bash
-
-python3 -m fabric_cf.orchestrator
+docker exec -i orchestrator-db psql orchestrator fabric < psql.upgrade

psql.upgrade

@@ -0,0 +1,2 @@
+ALTER TABLE "Slices" ADD COLUMN IF NOT EXISTS project_id VARCHAR;
+ALTER TABLE "Reservations" ADD COLUMN IF NOT EXISTS project_id VARCHAR;

requirements.txt

@@ -13,4 +13,4 @@ typing-extensions==3.7.4.3
 six==1.15.0
 wrapt==1.12.1
 fabric-message-bus==1.2.3
-fabric-fim==1.2.2
+fabric-fim==1.2.4