Reuse SpeechRecognizer and stream for multiple audio streams?

Hi team, is there any best practice on how to reuse the SpeechRecognizer for stream recognizing user audios? In our application, we know where user start talking and end talking so we can signal speech recognizer for it.

The reason I wanted to reuse SpeechRecognizer is because upon load testing on production, creating separate SpeechRecognizer consumes a lot of CPU and memory for websocket connections.

I am trying out the below code to test but it doesn't work since SpeechRecognizer seems to wait for stream to close in order to stop. However I don't want to close stream since it will need to recreate another connection later on.

import os
import random

import azure.cognitiveservices.speech as speechsdk
from gevent.event import Event

class StreamingSession:
    def __init__(self):
        self.transcription = ""
        self.closed = False
        self.recognition_done = Event()
        self.stream = speechsdk.audio.PushAudioInputStream()

        self.__register()
        self.speech_recognizer.start_continuous_recognition_async()

    def reset(self):
        self.transcription = ""
        self.closed = False
        self.recognition_done.clear()
        self.speech_recognizer.start_continuous_recognition_async()

    def add_chunk(self, chunk):
        if not self.closed:
            self.stream.write(chunk)

    def get_transcription(self):
        if not self.closed:
            self.recognition_done.wait()
            self.speech_recognizer.stop_continuous_recognition_async()
        return self.transcription

    def __register(self):
        # Initialize the speech configuration using environment variables
        selected_key = random.choice(self.speech_keys)
        self.speech_config = speechsdk.SpeechConfig(subscription=selected_key, 
                                                    region=os.environ.get('AZURE_STT_REGION'))
        self.audio_config = speechsdk.audio.AudioConfig(stream=self.stream)
        self.speech_recognizer = speechsdk.SpeechRecognizer(speech_config=self.speech_config, audio_config=self.audio_config)
        self.__connect_callbacks()

    def __connect_callbacks(self):
        self.speech_recognizer.recognized.connect(self.__recognized_callback)
        self.speech_recognizer.session_stopped.connect(self.__session_stopped_callback)

    def __recognized_callback(self, evt):
        self.transcription += evt.result.text

    def __session_stopped_callback(self, evt):
        self.recognition_done.set()
        self.closed = True
        print("Session is stopped")

    def __stream_recognize(self):
        self.recognition_done.wait()
        self.speech_recognizer.stop_continuous_recognition_async()

Ideally we want to reuse the same SpeechRecognizer for the same user throughout the session to recognize multiple audio streams. If that's not an option, we might have to create new SpeechRecognizer for every audio stream which is not performant, if we do clean up by disconnecting all callbacks, those operations are taking a long time > 10s. Please advise! Thank you!