KG portal generation operation

.gitignore

@@ -1,3 +1,4 @@
 __pycache__
 src/web_algebra/operations/.DS_Store
 /src/WebAlgebra.egg-info
+/packages

examples/generate-northwind-portal.json

@@ -0,0 +1,42 @@
+{
+  "@op": "GeneratePortal",
+  "args": {
+    "endpoint": {
+      "@op": "URI",
+      "args": {
+        "input": "https://linkeddatahub.com/demo/northwind-traders/sparql"
+      }
+    },
+    "package_name": "northwind-portal",
+    "package_namespace": {
+      "@op": "URI",
+      "args": {
+        "input": "https://localhost:4443/ns/northwind#"
+      }
+    },
+    "admin_base": {
+      "@op": "URI",
+      "args": {
+        "input": "https://admin.localhost:4443/"
+      }
+    },
+    "parent_container": {
+      "@op": "URI",
+      "args": {
+        "input": "https://localhost:4443/"
+      }
+    },
+    "files_container": {
+      "@op": "URI",
+      "args": {
+        "input": "https://localhost:4443/files/"
+      }
+    },
+    "services_container": {
+      "@op": "URI",
+      "args": {
+        "input": "https://localhost:4443/services/"
+      }
+    }
+  }
+}

pyproject.toml

@@ -11,6 +11,7 @@ dependencies = [
     "openai",
     "mcp[cli]==1.10.1",
     "pydantic-settings",
+    "lxml>=6.0.2",
 ]
 
 [build-system]

src/web_algebra/client.py

@@ -1,7 +1,11 @@
 from typing import Optional
 import ssl
 import json
+import logging
+import time
 import urllib.request
+import urllib.error
+from email.utils import parsedate_to_datetime
 from http.client import HTTPResponse
 from rdflib import Graph
 from rdflib.plugins.sparql.parser import parseQuery
@@ -67,6 +71,56 @@ def __init__(
             )
         ]
 
+    def _request_with_retry(self, request: urllib.request.Request, max_retries: int = 5) -> HTTPResponse:
+        """
+        Execute HTTP request with automatic retry on 429 (Too Many Requests) responses.
+
+        Respects the Retry-After header if present, otherwise uses exponential backoff.
+        All other HTTP errors are raised immediately without retry.
+
+        :param request: The urllib Request object to execute
+        :param max_retries: Maximum number of retry attempts (default 5)
+        :return: HTTPResponse object on success
+        :raises: urllib.error.HTTPError for non-429 errors or after max retries
+        """
+        attempt = 0
+
+        while attempt <= max_retries:
+            try:
+                return self.opener.open(request)
+            except urllib.error.HTTPError as e:
+                # Only retry on 429 (Too Many Requests)
+                if e.code != 429:
+                    raise
+
+                # Check if we've exhausted retries
+                if attempt >= max_retries:
+                    logging.error(f"Max retries ({max_retries}) exceeded for {request.full_url}")
+                    raise
+
+                # Parse Retry-After header
+                retry_after = e.headers.get('Retry-After')
+                if retry_after:
+                    try:
+                        # Try parsing as seconds (integer)
+                        wait_time = int(retry_after)
+                    except ValueError:
+                        # Try parsing as HTTP-date
+                        try:
+                            retry_date = parsedate_to_datetime(retry_after)
+                            wait_time = (retry_date - parsedate_to_datetime(time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()))).total_seconds()
+                            wait_time = max(0, wait_time)  # Ensure non-negative
+                        except Exception:
+                            # Fallback to exponential backoff if parsing fails
+                            wait_time = min(1 * (2 ** attempt), 60)
+                else:
+                    # No Retry-After header, use exponential backoff
+                    wait_time = min(1 * (2 ** attempt), 60)
+
+                attempt += 1
+                logging.warning(f"HTTP 429 received for {request.full_url}. Retry {attempt}/{max_retries} after {wait_time:.1f} seconds")
+                time.sleep(wait_time)
+
     def get(self, url: str) -> Graph:
         """
         Fetches RDF data from the given URL and returns it as an RDFLib Graph.
@@ -79,8 +133,8 @@ def get(self, url: str) -> Graph:
         headers = {"Accept": accept_header}
         request = urllib.request.Request(url, headers=headers)
 
-        # Perform the HTTP request
-        response = self.opener.open(request)
+        # Perform the HTTP request with retry on 429
+        response = self._request_with_retry(request)
 
         # Read and decode the response data
         data = response.read().decode("utf-8")
@@ -114,7 +168,7 @@ def post(self, url: str, graph: Graph) -> HTTPResponse:
             url, data=data.encode("utf-8"), headers=headers, method="POST"
         )
 
-        return self.opener.open(request)
+        return self._request_with_retry(request)
 
     def put(self, url: str, graph: Graph) -> HTTPResponse:
         """
@@ -134,7 +188,7 @@ def put(self, url: str, graph: Graph) -> HTTPResponse:
             url, data=data.encode("utf-8"), headers=headers, method="PUT"
         )
 
-        return self.opener.open(request)
+        return self._request_with_retry(request)
 
     def delete(self, url: str) -> HTTPResponse:
         """
@@ -145,7 +199,7 @@ def delete(self, url: str) -> HTTPResponse:
         """
         request = urllib.request.Request(url, method="DELETE")
 
-        return self.opener.open(request)
+        return self._request_with_retry(request)
 
     def patch(self, url: str, sparql_update: str) -> HTTPResponse:
         """
@@ -163,7 +217,7 @@ def patch(self, url: str, sparql_update: str) -> HTTPResponse:
             url, data=sparql_update.encode("utf-8"), headers=headers, method="PATCH"
         )
 
-        return self.opener.open(request)
+        return self._request_with_retry(request)
 
 
 class SPARQLClient:

src/web_algebra/operations/linkeddatahub/content/generate_class_containers.py

@@ -3,26 +3,27 @@
 from rdflib.namespace import RDF, RDFS, XSD, DCTERMS
 from rdflib.query import Result
 from web_algebra.operation import Operation
-from web_algebra.operations.linkeddatahub.create_container import CreateContainer
+from web_algebra.operations.linkeddatahub.create_item import CreateItem
 from web_algebra.operations.linked_data.post import POST
-from web_algebra.operations.linkeddatahub.add_generic_service import AddGenericService
+from web_algebra.operations.linkeddatahub.content.add_object_block import AddObjectBlock
 from web_algebra.json_result import JSONResult
 
 
 class GenerateClassContainers(Operation):
-    """Creates LinkedDataHub containers for ontology classes with instance list views.
+    """Creates LinkedDataHub items for ontology classes with instance list views.
 
     For each class in the ontology:
-    1. Creates a container using CreateContainer operation
+    1. Creates an item using CreateItem operation
     2. POSTs a SPIN sp:Select query that lists all instances
     3. POSTs an ldh:View that displays the instances
+    4. Adds an object block to display the view in the item
 
     This operation orchestrates actual HTTP operations to set up the portal structure.
     """
 
     @classmethod
     def description(cls) -> str:
-        return "Creates LinkedDataHub containers with instance list views for ontology classes"
+        return "Creates LinkedDataHub items with instance list views for ontology classes"
 
     @classmethod
     def inputSchema(cls) -> dict:
@@ -40,21 +41,26 @@ def inputSchema(cls) -> dict:
                 "endpoint": {
                     "type": "string",
                     "description": "SPARQL endpoint URI to be used by the queries"
+                },
+                "service_uri": {
+                    "type": "string",
+                    "description": "URI of the SPARQL service resource to reference in queries and views"
                 }
             },
-            "required": ["ontology", "parent_container", "endpoint"],
+            "required": ["ontology", "parent_container", "endpoint", "service_uri"],
         }
 
-    def execute(self, ontology: Graph, parent_container: URIRef, endpoint: URIRef) -> Result:
-        """Create LDH containers for ontology classes
+    def execute(self, ontology: Graph, parent_container: URIRef, endpoint: URIRef, service_uri: URIRef) -> Result:
+        """Create LDH items for ontology classes
 
         Args:
             ontology: RDF graph containing classes
-            parent_container: URI of parent container where class containers will be created
-            endpoint: SPARQL endpoint URI to be used by the queries
+            parent_container: URI of parent container where class items will be created
+            endpoint: SPARQL endpoint URI to be used by the queries (for query text generation)
+            service_uri: URI of the global SPARQL service resource to reference in queries and views
 
         Returns:
-            Concatenated Result containing all operation results (CreateContainer + AddGenericService + POST bindings)
+            Concatenated Result containing all operation results (CreateItem + POST + AddObjectBlock bindings)
         """
         # Define namespaces
         LDH = Namespace("https://w3id.org/atomgraph/linkeddatahub#")
@@ -92,56 +98,54 @@ def execute(self, ontology: Graph, parent_container: URIRef, endpoint: URIRef) -
             # Extract local name for URI
             class_local = self._get_local_name(class_uri)
 
-            logging.info(f"Creating container for class {class_uri}")
+            logging.info(f"Creating item for class {class_uri}")
 
-            # Step 1: Create container
+            # Step 1: Create item
             title = Literal(f"{class_local} instances", datatype=XSD.string)
             slug = Literal(class_local, datatype=XSD.string)
 
-            create_result = CreateContainer(settings=self.settings, context=self.context).execute(
+            create_result = CreateItem(settings=self.settings, context=self.context).execute(
                 parent_container, title, slug
             )
 
-            # Collect bindings from CreateContainer
+            # Collect bindings from CreateItem
             all_bindings.extend(create_result.bindings)
             all_vars.update(create_result.vars)
 
-            # Extract created container URL from result
-            container_uri = URIRef(create_result.bindings[0]["url"])
-            logging.info(f"Created container at {container_uri}")
-
-            # Step 2: Create service resource in the container
-            service_fragment = "Service"
-            service_result = AddGenericService(settings=self.settings, context=self.context).execute(
-                url=container_uri,
-                endpoint=endpoint,
-                title=Literal("SPARQL Service", datatype=XSD.string),
-                fragment=Literal(service_fragment, datatype=XSD.string)
-            )
-            all_bindings.extend(service_result.bindings)
-            all_vars.update(service_result.vars)
+            # Extract created item URL from result
+            item_uri = URIRef(create_result.bindings[0]["url"])
+            logging.info(f"Created item at {item_uri}")
 
-            # Service URI is container + fragment
-            service_uri = URIRef(f"{container_uri}#{service_fragment}")
-            logging.info(f"Created service resource at {service_uri}")
-
-            # Step 3: Create and POST sp:Select query
-            query_uri = URIRef(f"{container_uri}#Instances_Query")
+            # Step 2: Create and POST sp:Select query
+            query_uri = URIRef(f"{item_uri}#Instances_Query")
             sparql_text = self._generate_instance_query(class_uri)
 
             query_graph = self._build_query_graph(query_uri, class_local, sparql_text, service_uri, LDH, SP)
-            post_query_result = POST(settings=self.settings, context=self.context).execute(container_uri, query_graph)
+            post_query_result = POST(settings=self.settings, context=self.context).execute(item_uri, query_graph)
             all_bindings.extend(post_query_result.bindings)
             all_vars.update(post_query_result.vars)
-            logging.info(f"Posted query to {container_uri}")
+            logging.info(f"Posted query to {item_uri}")
 
-            # Step 4: Create and POST ldh:View
-            view_uri = URIRef(f"{container_uri}#Instances_View")
-            view_graph = self._build_view_graph(view_uri, class_local, query_uri, LDH, SP, AC, SPIN)
-            post_view_result = POST(settings=self.settings, context=self.context).execute(container_uri, view_graph)
+            # Step 3: Create and POST ldh:View with service reference
+            view_uri = URIRef(f"{item_uri}#Instances_View")
+            view_graph = self._build_view_graph(view_uri, class_local, query_uri, service_uri, LDH, SP, AC, SPIN)
+            post_view_result = POST(settings=self.settings, context=self.context).execute(item_uri, view_graph)
             all_bindings.extend(post_view_result.bindings)
             all_vars.update(post_view_result.vars)
-            logging.info(f"Posted view to {container_uri}")
+            logging.info(f"Posted view to {item_uri}")
+
+            # Step 4: Add object block to display the view in the item
+            logging.info(f"Adding object block to display view in {item_uri}")
+            add_block_result = AddObjectBlock(settings=self.settings, context=self.context).execute(
+                url=item_uri,
+                value=view_uri,
+                title=Literal(f"All {class_local}", datatype=XSD.string),
+                fragment=Literal("InstancesBlock", datatype=XSD.string),
+                mode=None
+            )
+            all_bindings.extend(add_block_result.bindings)
+            all_vars.update(add_block_result.vars)
+            logging.info(f"Added object block to {item_uri}")
 
         # Create concatenated Result using JSONResult
         return JSONResult(list(all_vars), all_bindings)
@@ -184,7 +188,7 @@ def _build_query_graph(self, query_uri: URIRef, class_local: str, sparql_text: s
         return g
 
     def _build_view_graph(self, view_uri: URIRef, class_local: str, query_uri: URIRef,
-                         LDH, SP, AC, SPIN) -> Graph:
+                         service_uri: URIRef, LDH, SP, AC, SPIN) -> Graph:
         """Build RDF graph for ldh:View resource"""
         g = Graph()
         g.bind("ldh", LDH)
@@ -195,6 +199,7 @@ def _build_view_graph(self, view_uri: URIRef, class_local: str, query_uri: URIRe
         g.add((view_uri, RDF.type, LDH.View))
         g.add((view_uri, DCTERMS.title, Literal(f"All {class_local}")))
         g.add((view_uri, SPIN.query, query_uri))
+        g.add((view_uri, LDH.service, service_uri))
         g.add((view_uri, AC.mode, AC.ListMode))
 
         return g
@@ -231,4 +236,14 @@ def execute_json(self, arguments: dict, variable_stack: list = []) -> Result:
                 f"GenerateClassContainers operation expects 'endpoint' to be URIRef, got {type(endpoint_data)}"
             )
 
-        return self.execute(ontology_data, parent_container_data, endpoint_data)
+        # Process service_uri
+        service_uri_data = Operation.process_json(
+            self.settings, arguments["service_uri"], self.context, variable_stack
+        )
+
+        if not isinstance(service_uri_data, URIRef):
+            raise TypeError(
+                f"GenerateClassContainers operation expects 'service_uri' to be URIRef, got {type(service_uri_data)}"
+            )
+
+        return self.execute(ontology_data, parent_container_data, endpoint_data, service_uri_data)