Merge pull request #564 from a10pepo/kafka

entrega-kafka
a10pepo · Feb 15, 2025 · cbb2017 · cbb2017
2 parents 96ac2fe + de32ef5
commit cbb2017
Show file tree

Hide file tree

Showing 5 changed files with 368 additions and 0 deletions.
diff --git a/ALUMNOS/MDAA/MAR_SOLER/KAFKA/consumer.py b/ALUMNOS/MDAA/MAR_SOLER/KAFKA/consumer.py
@@ -0,0 +1,31 @@
+from confluent_kafka import Consumer, KafkaError
+
+config = {
+    'bootstrap.servers': 'localhost:9092',  
+    'group.id': 'python-consumer-group',
+    'auto.offset.reset': 'earliest'  
+}
+
+consumer = Consumer(config)
+
+topic = "recetas"
+consumer.subscribe([topic])
+
+try:
+    while True:
+        msg = consumer.poll(1.0)  # Lee nuevos mensajes cada 1 segundo
+
+        if msg is None:
+            continue
+        if msg.error():
+            if msg.error().code() == KafkaError._PARTITION_EOF:
+                print("No hay más mensajes en esta partición.")
+            else:
+                print("Error al recibir mensaje: {}".format(msg.error()))
+        else:
+            print("Nuevo mensaje: {}".format(msg.value().decode("utf-8")))
+
+except KeyboardInterrupt:
+    pass
+finally:
+    consumer.close()
diff --git a/ALUMNOS/MDAA/MAR_SOLER/KAFKA/docker-compose.yml b/ALUMNOS/MDAA/MAR_SOLER/KAFKA/docker-compose.yml
@@ -0,0 +1,150 @@
+---
+version: '3.5'
+services:
+  zookeeper:
+    image: confluentinc/cp-zookeeper:latest
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+    ports:
+      - 2181:2181
+  kafka: 
+    image: confluentinc/cp-kafka:7.3.3
+    depends_on:
+      - zookeeper
+    ports:
+      - 9092:9092
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+  kafka-ui:
+    image: provectuslabs/kafka-ui:latest
+    container_name: kafka-ui-3
+    depends_on:
+      - kafka
+    ports:
+      - "8080:8080"
+    restart: always
+    environment:
+      - KAFKA_CLUSTERS_0_NAME=base
+      - KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=kafka:29092
+  ksql-server:
+    image: confluentinc/cp-ksql-server:latest
+    depends_on:
+      - kafka
+    ports:
+      - 8088:8088
+    environment:
+      KSQL_BOOTSTRAP_SERVERS: kafka:29092
+      KSQL_LISTENERS: http://0.0.0.0:8088
+  ksql-cli:
+    image: confluentinc/cp-ksql-cli:latest
+    depends_on:
+      - ksql-server
+    entrypoint: /bin/sh
+    tty: true
+
+  kafka-connect:
+    image: confluentinc/cp-kafka-connect:7.3.3
+    depends_on:
+      - kafka
+      - schema-registry
+      - postgres
+    environment:
+      CONNECT_PLUGIN_PATH: /usr/share/java,/usr/share/confluent-hub-components
+      CONNECT_BOOTSTRAP_SERVERS: kafka:29092
+      CONNECT_REST_ADVERTISED_HOST_NAME: connect
+      CONNECT_GROUP_ID: compose-connect-group
+      CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
+      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
+      CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
+      CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
+      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
+      CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
+      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
+      CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
+      CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.storage.StringConverter
+      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
+      # CLASSPATH required due to CC-2422
+      CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-7.3.3.jar
+      CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"
+      CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"
+      CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR
+    command:
+      - bash
+      - -c
+      - |
+        confluent-hub install confluentinc/kafka-connect-jdbc:10.7.4
+        /etc/confluent/docker/run
+
+  schema-registry:
+    image: confluentinc/cp-schema-registry:7.3.3
+    hostname: schema-registry
+    container_name: schema-registry
+    depends_on:
+      - kafka
+    ports:
+      - "8081:8081"
+    environment:
+      SCHEMA_REGISTRY_HOST_NAME: schema-registry
+      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'kafka:29092'
+      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
+
+
+  control-center:
+    image: confluentinc/cp-enterprise-control-center:7.3.3
+    hostname: control-center
+    container_name: control-center
+    depends_on:
+      - kafka
+      - schema-registry
+      - kafka-connect
+      - ksql-server
+    ports:
+      - "9021:9021"
+    environment:
+      CONTROL_CENTER_BOOTSTRAP_SERVERS: 'kafka:29092'
+      CONTROL_CENTER_CONNECT_CONNECT-DEFAULT_CLUSTER: 'kafka-connect:8083'
+      CONTROL_CENTER_KSQL_KSQLDB1_URL: "http://ksql-server:8088"
+      CONTROL_CENTER_KSQL_KSQLDB1_ADVERTISED_URL: "http://localhost:8088"
+      CONTROL_CENTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
+      CONTROL_CENTER_REPLICATION_FACTOR: 1
+      CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1
+      CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1
+      CONFLUENT_METRICS_TOPIC_REPLICATION: 1
+      CONTROL_CENTER_CONNECT_HEALTHCHECK_ENDPOINT: '/connectors'
+      PORT: 9021
+
+  postgres:
+    container_name: postgres_container
+    image: postgres:12.1
+    environment:
+      POSTGRES_USER: ${POSTGRES_USER:-postgres}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-Welcome01}
+      PGDATA: /data/postgres
+    volumes:
+      - postgres:/data/postgres
+    ports:
+      - "5432:5432"
+    restart: unless-stopped
+
+  pgadmin:
+      container_name: pgadmin_container
+      image: dpage/pgadmin4:4.16
+      environment:
+        PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL:[email protected]}
+        PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD:-admin}
+      volumes:
+        - pgadmin:/root/.pgadmin
+      ports:
+        - "${PGADMIN_PORT:-5050}:80"
+      restart: unless-stopped
+
+
+volumes:
+  postgres:
+  pgadmin:
diff --git a/ALUMNOS/MDAA/MAR_SOLER/KAFKA/pom.xml b/ALUMNOS/MDAA/MAR_SOLER/KAFKA/pom.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.gft.dlp.kafka</groupId>
+    <artifactId>lab03</artifactId>
+    <version>1.0-SNAPSHOT</version>
+
+    <properties>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
+    </properties>
+
+    <dependencies>
+        <!-- https://mvnrepository.com/artifact/org.apache.kafka/kafka -->
+        <dependency>
+            <groupId>org.apache.kafka</groupId>
+            <artifactId>kafka_2.13</artifactId>
+            <version>2.4.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.kafka</groupId>
+            <artifactId>kafka-streams</artifactId>
+            <version>2.4.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.github.javafaker</groupId>
+            <artifactId>javafaker</artifactId>
+            <version>1.0.2</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/io.confluent/common-utils -->
+        <dependency>
+            <groupId>io.confluent</groupId>
+            <artifactId>common-utils</artifactId>
+            <version>4.1.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>1.7.5</version>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-log4j12</artifactId>
+            <version>1.7.5</version>
+        </dependency>
+    </dependencies>
+    <repositories>
+        <repository>
+            <id>confluent</id>
+            <url>https://packages.confluent.io/maven/</url>
+        </repository>
+    </repositories>
+</project>
diff --git a/ALUMNOS/MDAA/MAR_SOLER/KAFKA/producer.py b/ALUMNOS/MDAA/MAR_SOLER/KAFKA/producer.py
@@ -0,0 +1,31 @@
+import time
+from json import dumps
+from confluent_kafka import Producer
+import re
+
+config = {
+    'bootstrap.servers': 'localhost:9092',  
+    'client.id': 'python-producer'
+}
+
+producer = Producer(config)
+
+topic_kafka = "recetas"
+
+file1 = open("recetas.txt",encoding="utf8")
+Lines = file1.readlines()
+
+count = 0
+
+for line in Lines:
+    time.sleep(2)
+    print( line.strip() + "\n")
+    words = re.findall(r"[\w']+|[.,!?;]", line)
+    for word in words:
+        data_bytes = word  
+        key = str(count)
+        producer.produce(topic=topic_kafka, value=data_bytes, key=key)
+        producer.flush()
+
+if producer.flush() != 0:
+    print("Error en el envío de algunos mensajes.")
diff --git a/ALUMNOS/MDAA/MAR_SOLER/KAFKA/recetas.txt b/ALUMNOS/MDAA/MAR_SOLER/KAFKA/recetas.txt
@@ -0,0 +1,100 @@
+Paella Valenciana
+Tacos al Pastor
+Enchiladas Verdes
+Pizza Margarita
+Sushi de Salmón
+Hamburguesa Clásica
+Lasagna Boloñesa
+Ceviche Peruano
+Empanadas de Carne
+Falafel con Hummus
+Gazpacho Andaluz
+Moussaka Griega
+Pad Thai
+Ratatouille
+Biryani de Pollo
+Chili con Carne
+Croquetas de Jamón
+Pollo a la Parrilla
+Sopa de Tomate
+Risotto de Champiñones
+Filete de Ternera
+Pescado a la Plancha
+Ensalada César
+Tortilla Española
+Arroz con Leche
+Sopa de Mariscos
+Pollo al Ajillo
+Pasta Carbonara
+Pan de Ajo
+Curry Vegetariano
+Guiso de Lentejas
+Empanadas de Pollo
+Pollo a la Cacciatora
+Fajitas de Res
+Bacalao a la Vizcaína
+Crema de Calabaza
+Quiche Lorraine
+Macarrones con Queso
+Burrito Mexicano
+Sándwich Cubano
+Tarta de Queso
+Churros con Chocolate
+Pollo al Curry
+Arroz Frito
+Sushi de Atún
+Pato a la Naranja
+Minestrone
+Canelones de Espinacas
+Mousse de Chocolate
+Panna Cotta
+Pisto Manchego
+Croissant de Jamón y Queso
+Sopa de Ajo
+Goulash Húngaro
+Bruschetta Italiana
+Focaccia de Romero
+Paella Mixta
+Pollo a la Barbacoa
+Carne Asada
+Chili Vegetariano
+Lasaña de Verduras
+Sushi Vegetariano
+Bocadillo de Calamares
+Ensalada Caprese
+Steak Tartare
+Pollo al Horno
+Crepes Suzette
+Pollo al Chipotle
+Pastel de Carne
+Tortilla de Patatas
+Patatas Bravas
+Salmón al Horno
+Albóndigas en Salsa
+Ensalada Griega
+Calamares a la Romana
+Pechuga de Pollo Rellena
+Ensalada de Quinoa
+Pasta al Pesto
+Croquetas de Pollo
+Fideos de Arroz con Verduras
+Empanada Gallega
+Tiramisú
+Panqueques Americanos
+Chili de Quinoa
+Brócoli al Ajillo
+Nuggets de Pollo Caseros
+Puré de Papas
+Crema de Champiñones
+Pizza Pepperoni
+Pollo con Verduras
+Espaguetis al Pesto
+Ensalada Mixta
+Canelones de Carne
+Gambas al Ajillo
+Risotto de Mariscos
+Merluza en Salsa Verde
+Ensalada de Pasta
+Pechuga a la Plancha
+Lomo Saltado
+Pastel de Chocolate